Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[remerge] A more spec-compliant/resilient OCI distribution implementation #318

Merged
merged 18 commits into from
Dec 7, 2022
Merged

[remerge] A more spec-compliant/resilient OCI distribution implementation #318

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
68d50ca
setup reference implementation registry for testing
joshspicer Dec 2, 2022
ebfc4c3
uncomment
joshspicer Dec 2, 2022
68ae246
revert plainHTTP defaulting to true
joshspicer Dec 2, 2022
1945816
mode for plain http when targetting localhost
joshspicer Dec 2, 2022
2701867
print only when plain HTTP
joshspicer Dec 2, 2022
c6f673c
surface JSON errors better
joshspicer Dec 2, 2022
ec68d10
refactor out reading in tgz Buffer to only do that work once
joshspicer Dec 2, 2022
1d5ed62
generate a postUploadSessionId for each blob that needs to be uploaded
joshspicer Dec 2, 2022
20d7774
update auth contract change in utils file
joshspicer Dec 2, 2022
28dd163
factor out containerFeaturesOCIPush test to its own matrix run
joshspicer Dec 2, 2022
de93dad
clarifying comment
joshspicer Dec 2, 2022
b27f9fc
add comments clarifying auth
joshspicer Dec 2, 2022
56ceefc
allow localhost past the domain name check
joshspicer Dec 2, 2022
866ce3e
add 'info tags' and 'info manifest' commands'
joshspicer Dec 2, 2022
227a85e
use new info commands in test
joshspicer Dec 3, 2022
f80ba58
typecheck
joshspicer Dec 3, 2022
cdd92f5
clarify error case of getManifest()
joshspicer Dec 3, 2022
fd4de3f
Apply suggestions from code review
joshspicer Dec 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
setup reference implementation registry for testing
  • Loading branch information
@joshspicer
joshspicer committed Dec 2, 2022
commit 68d50ca975e4a64d9b5dfae3f650d3d4e2b7c3b5
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ logs
*.tgz
tmp
tmp[0-9]
tmp/*
build-tmp
.DS_Store
.env
Expand Down
158 changes: 97 additions & 61 deletions src/spec-configuration/containerCollectionsOCI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers';
export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar';
export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devcontainers.collection.layer.v1+json';

export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string };
export type HEADERS = { 'authorization'?: string; 'user-agent': string; 'content-type'?: string; 'accept'?: string; 'content-length'?: string };

// Represents the unique OCI identifier for a Feature or Template.
// eg: ghcr.io/devcontainers/features/go:1.0.0
Expand Down Expand Up @@ -201,9 +201,9 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri
'accept': mimeType || 'application/vnd.oci.image.manifest.v1+json',
};

const auth = authToken ?? await fetchRegistryAuthToken(output, ref.registry, ref.path, env, 'pull');
if (auth) {
headers['authorization'] = `Bearer ${auth}`;
const authorization = authToken ?? await fetchAuthorization(output, ref.registry, ref.path, env, 'pull');
if (authorization) {
headers['authorization'] = authorization;
}

const options = {
Expand All @@ -221,72 +221,108 @@ export async function getManifest(output: Log, env: NodeJS.ProcessEnv, url: stri
}
}

// https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74
export async function fetchRegistryAuthToken(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string): Promise<string | undefined> {
if (registry === 'mcr.microsoft.com') {
return undefined;
}

const headers: HEADERS = {
'user-agent': 'devcontainer'
};
async function getBasicAuthCredential(output: Log, registry: string, env: NodeJS.ProcessEnv): Promise<string | undefined> {
// TODO: Also read OS keychain/docker config for auth in various registries!

// TODO: Read OS keychain/docker config for auth in various registries!

let userToken = '';
let userToken: string | undefined = undefined;
if (!!env['GITHUB_TOKEN'] && registry === 'ghcr.io') {
userToken = env['GITHUB_TOKEN'];
} else if (!!env['DEVCONTAINERS_OCI_AUTH']) {
// eg: DEVCONTAINERS_OCI_AUTH=domain1:token1,domain2:token2
// eg: DEVCONTAINERS_OCI_AUTH=domain1|user1|token1,domain2|user2|token2
const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(',');
const authContext = authContexts.find(a => a.split(':')[0] === registry);
if (authContext && authContext.length === 2) {
userToken = authContext.split(':')[1];
const authContext = authContexts.find(a => a.split('|')[0] === registry);

if (authContext) {
output.write(`Using match from DEVCONTAINERS_OCI_AUTH for registry '${registry}'`, LogLevel.Trace);
const split = authContext.split('|');
userToken = `${split[1]}:${split[2]}`;
}
} else {
output.write('No oauth authentication credentials found.', LogLevel.Trace);
}

if (userToken) {
const base64Encoded = Buffer.from(`USERNAME:${userToken}`).toString('base64');
headers['authorization'] = `Basic ${base64Encoded}`;
return Buffer.from(userToken).toString('base64');
}

const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry;
const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry;
const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`;
output.write(`url: ${url}`, LogLevel.Trace);

const options = {
type: 'GET',
url: url,
headers: headers
};
// Error
output.write(`No authentication credentials found for registry '${registry}'.`, LogLevel.Error);
return undefined;
}

let authReq: Buffer;
try {
authReq = await request(options, output);
} catch (e: any) {
output.write(`Failed to get registry auth token with error: ${e}`, LogLevel.Error);
return undefined;
// https://github.com/oras-project/oras-go/blob/97a9c43c52f9d89ecf5475bc59bd1f96c8cc61f6/registry/remote/auth/scope.go#L60-L74
// Some registries (eg: ghcr.io) expect a scoped token to target resources.

// async function generateScopeTokenCredential(output: Log, registry: string, ociRepoPath: string, env: NodeJS.ProcessEnv, operationScopes: string, basicAuthTokenBase64: string | undefined = undefined): Promise<string | undefined> {
// if (registry === 'mcr.microsoft.com') {
// return undefined;
// }

// const headers: HEADERS = {
// 'user-agent': 'devcontainer'
// };

// if (!basicAuthTokenBase64) {
// basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env);
// }

// headers['authorization'] = `Basic ${basicAuthTokenBase64}`;

// const authServer = registry === 'docker.io' ? 'auth.docker.io' : registry;
// const registryServer = registry === 'docker.io' ? 'registry.docker.io' : registry;
// const url = `https://${authServer}/token?scope=repository:${ociRepoPath}:${operationScopes}&service=${registryServer}`;
// output.write(`url: ${url}`, LogLevel.Trace);

// const options = {
// type: 'GET',
// url: url,
// headers: headers
// };

// let authReq: Buffer;
// try {
// authReq = await request(options, output);
// } catch (e: any) {
// output.write(`Unable to request scope token from registry ${registry}: ${e}`, LogLevel.Warning);
// return;
// }

// if (!authReq) {
// output.write('Failed to get registry auth token', LogLevel.Error);
// return undefined;
// }

// let scopeToken: string | undefined;
// try {
// scopeToken = JSON.parse(authReq.toString())?.token;
// } catch {
// // not JSON
// }
// if (!scopeToken) {
// output.write('Failed to parse registry auth token response', LogLevel.Error);
// return undefined;
// }

// return scopeToken;
// }


// Exported Function
// Will attempt to generate/fetch the correct authorization header for subsequent requests (Bearer or Basic)
export async function fetchAuthorization(output: Log, registry: string, _ociRepoPath: string, env: NodeJS.ProcessEnv, _operationScopes: string): Promise<string | undefined> {
const basicAuthTokenBase64 = await getBasicAuthCredential(output, registry, env);
// const scopeToken = generateScopeTokenCredential(output, registry, ociRepoPath, env, operationScopes, basicAuthTokenBase64);

// if (scopeToken) {
// output.write(`Using scope token for registry '${registry}'`, LogLevel.Trace);
// return `Bearer ${scopeToken}`;
// }

if (basicAuthTokenBase64) {
output.write(`Using basic auth token for registry '${registry}'`, LogLevel.Trace);
return `Basic ${basicAuthTokenBase64}`;
}

if (!authReq) {
output.write('Failed to get registry auth token', LogLevel.Error);
return undefined;
}
return undefined;

let token: string | undefined;
try {
token = JSON.parse(authReq.toString())?.token;
} catch {
// not JSON
}
if (!token) {
output.write('Failed to parse registry auth token response', LogLevel.Error);
return undefined;
}
return token;
}

// Lists published versions/tags of a feature/template
Expand All @@ -295,17 +331,17 @@ export async function getPublishedVersions(ref: OCIRef, output: Log, sorted: boo
try {
const url = `https://${ref.registry}/v2/${ref.namespace}/${ref.id}/tags/list`;

let authToken = await fetchRegistryAuthToken(output, ref.registry, ref.path, process.env, 'pull');
let authorization = await fetchAuthorization(output, ref.registry, ref.path, process.env, 'pull');

if (!authToken) {
if (!authorization) {
output.write(`(!) ERR: Failed to get published versions for ${collectionType}: ${ref.resource}`, LogLevel.Error);
return undefined;
}

const headers: HEADERS = {
'user-agent': 'devcontainer',
'accept': 'application/json',
'authorization': `Bearer ${authToken}`
'authorization': authorization
};

const options = {
Expand Down Expand Up @@ -352,9 +388,9 @@ export async function getBlob(output: Log, env: NodeJS.ProcessEnv, url: string,
'accept': 'application/vnd.oci.image.manifest.v1+json',
};

const auth = authToken ?? await fetchRegistryAuthToken(output, ociRef.registry, ociRef.path, env, 'pull');
if (auth) {
headers['authorization'] = `Bearer ${auth}`;
const authorization = authToken ?? await fetchAuthorization(output, ociRef.registry, ociRef.path, env, 'pull');
if (authorization) {
headers['authorization'] = authorization;
}

const options = {
Expand Down
Loading