This repository has been archived by the owner on Sep 9, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1k
/
source.go
399 lines (343 loc) · 10.9 KB
/
source.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
package gps
import (
"fmt"
"sync"
)
type source interface {
syncLocal() error
checkExistence(sourceExistence) bool
exportVersionTo(Version, string) error
getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
listPackages(ProjectRoot, Version) (PackageTree, error)
listVersions() ([]Version, error)
revisionPresentIn(Revision) (bool, error)
}
type sourceMetaCache struct {
//Version string // TODO(sdboyer) use this
infos map[Revision]projectInfo
ptrees map[Revision]PackageTree
vMap map[UnpairedVersion]Revision
rMap map[Revision][]UnpairedVersion
// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
}
// projectInfo holds manifest and lock
type projectInfo struct {
Manifest
Lock
}
type existence struct {
// The existence levels for which a search/check has been performed
s sourceExistence
// The existence levels verified to be present through searching
f sourceExistence
}
func newMetaCache() *sourceMetaCache {
return &sourceMetaCache{
infos: make(map[Revision]projectInfo),
ptrees: make(map[Revision]PackageTree),
vMap: make(map[UnpairedVersion]Revision),
rMap: make(map[Revision][]UnpairedVersion),
}
}
type baseVCSSource struct {
// Object for the cache repository
crepo *repo
// Indicates the extent to which we have searched for, and verified, the
// existence of the project/repo.
ex existence
// ProjectAnalyzer used to fulfill getManifestAndLock
an ProjectAnalyzer
// The project metadata cache. This is (or is intended to be) persisted to
// disk, for reuse across solver runs.
dc *sourceMetaCache
// lvfunc allows the other vcs source types that embed this type to inject
// their listVersions func into the baseSource, for use as needed.
lvfunc func() (vlist []Version, err error)
// lock to serialize access to syncLocal
synclock sync.Mutex
// Globalish flag indicating whether a "full" sync has been performed. Also
// used as a one-way gate to ensure that the full syncing routine is never
// run more than once on a given source instance.
allsync bool
// The error, if any, that occurred on syncLocal
syncerr error
// Whether the cache has the latest info on versions
cvsync bool
}
func (bs *baseVCSSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
if err := bs.ensureCacheExistence(); err != nil {
return nil, nil, err
}
rev, err := bs.toRevOrErr(v)
if err != nil {
return nil, nil, err
}
// Return the info from the cache, if we already have it
if pi, exists := bs.dc.infos[rev]; exists {
return pi.Manifest, pi.Lock, nil
}
// Cache didn't help; ensure our local is fully up to date.
err = bs.syncLocal()
if err != nil {
return nil, nil, err
}
bs.crepo.mut.Lock()
// Always prefer a rev, if it's available
if pv, ok := v.(PairedVersion); ok {
err = bs.crepo.r.UpdateVersion(pv.Underlying().String())
} else {
err = bs.crepo.r.UpdateVersion(v.String())
}
bs.crepo.mut.Unlock()
if err != nil {
// TODO(sdboyer) More-er proper-er error
panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.crepo.r.LocalPath(), v.String(), unwrapVcsErr(err)))
}
bs.crepo.mut.RLock()
m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.r.LocalPath(), r)
// TODO(sdboyer) cache results
bs.crepo.mut.RUnlock()
if err == nil {
if l != nil {
l = prepLock(l)
}
// If m is nil, prepManifest will provide an empty one.
pi := projectInfo{
Manifest: prepManifest(m),
Lock: l,
}
bs.dc.infos[rev] = pi
return pi.Manifest, pi.Lock, nil
}
return nil, nil, unwrapVcsErr(err)
}
// toRevision turns a Version into a Revision, if doing so is possible based on
// the information contained in the version itself, or in the cache maps.
func (dc *sourceMetaCache) toRevision(v Version) Revision {
switch t := v.(type) {
case Revision:
return t
case PairedVersion:
return t.Underlying()
case UnpairedVersion:
// This will return the empty rev (empty string) if we don't have a
// record of it. It's up to the caller to decide, for example, if
// it's appropriate to update the cache.
return dc.vMap[t]
default:
panic(fmt.Sprintf("Unknown version type %T", v))
}
}
// toUnpaired turns a Version into an UnpairedVersion, if doing so is possible
// based on the information contained in the version itself, or in the cache
// maps.
//
// If the input is a revision and multiple UnpairedVersions are associated with
// it, whatever happens to be the first is returned.
func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
switch t := v.(type) {
case UnpairedVersion:
return t
case PairedVersion:
return t.Unpair()
case Revision:
if upv, has := dc.rMap[t]; has && len(upv) > 0 {
return upv[0]
}
return nil
default:
panic(fmt.Sprintf("unknown version type %T", v))
}
}
func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) {
// First and fastest path is to check the data cache to see if the rev is
// present. This could give us false positives, but the cases where that can
// occur would require a type of cache staleness that seems *exceedingly*
// unlikely to occur.
if _, has := bs.dc.infos[r]; has {
return true, nil
} else if _, has := bs.dc.rMap[r]; has {
return true, nil
}
err := bs.ensureCacheExistence()
if err != nil {
return false, err
}
bs.crepo.mut.RLock()
defer bs.crepo.mut.RUnlock()
return bs.crepo.r.IsReference(string(r)), nil
}
func (bs *baseVCSSource) ensureCacheExistence() error {
// Technically, methods could could attempt to return straight from the
// metadata cache even if the repo cache doesn't exist on disk. But that
// would allow weird state inconsistencies (cache exists, but no repo...how
// does that even happen?) that it'd be better to just not allow so that we
// don't have to think about it elsewhere
if !bs.checkExistence(existsInCache) {
if bs.checkExistence(existsUpstream) {
bs.crepo.mut.Lock()
if bs.crepo.synced {
// A second ensure call coming in while the first is completing
// isn't terribly unlikely, especially for a large repo. In that
// event, the synced flag will have flipped on by the time we
// acquire the lock. If it has, there's no need to do this work
// twice.
bs.crepo.mut.Unlock()
return nil
}
err := bs.crepo.r.Get()
if err != nil {
bs.crepo.mut.Unlock()
return fmt.Errorf("failed to create repository cache for %s with err:\n%s", bs.crepo.r.Remote(), unwrapVcsErr(err))
}
bs.crepo.synced = true
bs.ex.s |= existsInCache
bs.ex.f |= existsInCache
bs.crepo.mut.Unlock()
} else {
return fmt.Errorf("project %s does not exist upstream", bs.crepo.r.Remote())
}
}
return nil
}
// checkExistence provides a direct method for querying existence levels of the
// source. It will only perform actual searching (local fs or over the network)
// if no previous attempt at that search has been made.
//
// Note that this may perform read-ish operations on the cache repo, and it
// takes a lock accordingly. This makes it unsafe to call from a segment where
// the cache repo mutex is already write-locked, as deadlock will occur.
func (bs *baseVCSSource) checkExistence(ex sourceExistence) bool {
if bs.ex.s&ex != ex {
if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
panic("should now be implemented in bridge")
}
if ex&existsInCache != 0 && bs.ex.s&existsInCache == 0 {
bs.crepo.mut.RLock()
bs.ex.s |= existsInCache
if bs.crepo.r.CheckLocal() {
bs.ex.f |= existsInCache
}
bs.crepo.mut.RUnlock()
}
if ex&existsUpstream != 0 && bs.ex.s&existsUpstream == 0 {
bs.crepo.mut.RLock()
bs.ex.s |= existsUpstream
if bs.crepo.r.Ping() {
bs.ex.f |= existsUpstream
}
bs.crepo.mut.RUnlock()
}
}
return ex&bs.ex.f == ex
}
// syncLocal ensures the local data we have about the source is fully up to date
// with what's out there over the network.
func (bs *baseVCSSource) syncLocal() error {
// Ensure we only have one goroutine doing this at a time
bs.synclock.Lock()
defer bs.synclock.Unlock()
// ...and that we only ever do it once
if bs.allsync {
// Return the stored err, if any
return bs.syncerr
}
bs.allsync = true
// First, ensure the local instance exists
bs.syncerr = bs.ensureCacheExistence()
if bs.syncerr != nil {
return bs.syncerr
}
_, bs.syncerr = bs.lvfunc()
if bs.syncerr != nil {
return bs.syncerr
}
// This case is really just for git repos, where the lvfunc doesn't
// guarantee that the local repo is synced
if !bs.crepo.synced {
bs.crepo.mut.Lock()
err := bs.crepo.r.Update()
if err != nil {
bs.syncerr = fmt.Errorf("failed fetching latest updates with err: %s", unwrapVcsErr(err))
bs.crepo.mut.Unlock()
return bs.syncerr
}
bs.crepo.synced = true
bs.crepo.mut.Unlock()
}
return nil
}
func (bs *baseVCSSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
if err = bs.ensureCacheExistence(); err != nil {
return
}
var r Revision
if r, err = bs.toRevOrErr(v); err != nil {
return
}
// Return the ptree from the cache, if we already have it
var exists bool
if ptree, exists = bs.dc.ptrees[r]; exists {
return
}
// Not in the cache; check out the version and do the analysis
bs.crepo.mut.Lock()
// Check out the desired version for analysis
if r != "" {
// Always prefer a rev, if it's available
err = bs.crepo.r.UpdateVersion(string(r))
} else {
// If we don't have a rev, ensure the repo is up to date, otherwise we
// could have a desync issue
if !bs.crepo.synced {
err = bs.crepo.r.Update()
if err != nil {
err = fmt.Errorf("could not fetch latest updates into repository: %s", unwrapVcsErr(err))
return
}
bs.crepo.synced = true
}
err = bs.crepo.r.UpdateVersion(v.String())
}
if err == nil {
ptree, err = ListPackages(bs.crepo.r.LocalPath(), string(pr))
// TODO(sdboyer) cache errs?
if err == nil {
bs.dc.ptrees[r] = ptree
}
} else {
err = unwrapVcsErr(err)
}
bs.crepo.mut.Unlock()
return
}
// toRevOrErr makes all efforts to convert a Version into a rev, including
// updating the cache repo (if needed). It does not guarantee that the returned
// Revision actually exists in the repository (as one of the cheaper methods may
// have had bad data).
func (bs *baseVCSSource) toRevOrErr(v Version) (r Revision, err error) {
r = bs.dc.toRevision(v)
if r == "" {
// Rev can be empty if:
// - The cache is unsynced
// - A version was passed that used to exist, but no longer does
// - A garbage version was passed. (Functionally indistinguishable from
// the previous)
if !bs.cvsync {
// call the lvfunc to sync the meta cache
_, err = bs.lvfunc()
if err != nil {
return
}
}
r = bs.dc.toRevision(v)
// If we still don't have a rev, then the version's no good
if r == "" {
err = fmt.Errorf("version %s does not exist in source %s", v, bs.crepo.r.Remote())
}
}
return
}
func (bs *baseVCSSource) exportVersionTo(v Version, to string) error {
return bs.crepo.exportVersionTo(v, to)
}