Skip to content
Source
Sign inSign up
Open History to see the last commit

gitHubMediaFetch.ts

189 lines7.9 KB

1import type { SourceControlMediaRequest } from "@t3tools/source-control-core/server/driver";
2import * as Clock from "effect/Clock";
3import * as Effect from "effect/Effect";
4import * as Redacted from "effect/Redacted";
5import {
6 FetchHttpClient,
7 HttpClient,
8 HttpClientRequest,
9 type HttpClientResponse,
10} from "effect/http";
11​
12import { GITHUB_CREDENTIALED_MEDIA_HOSTS } from "../shared/gitHubMedia.ts";
13​
14import * as GitHubCredentials from "./GitHubCredentials.ts";
15import * as GitHubRest from "./GitHubRest.ts";
16​
17/**
18 * Everything a redirect leads to — the presigned object stores GitHub hands assets off to —
19 * authorizes with its own signature, and some of them reject a request that also carries a
20 * bearer token.
21 */
22const isCredentialedHost = (url: string) => {
23 const target = URL.parse(url);
24 return (
25 target !== null &&
26 target.protocol === "https:" &&
27 target.port === "" &&
28 GITHUB_CREDENTIALED_MEDIA_HOSTS.has(target.hostname.toLowerCase())
29 );
30};
31​
32const MAX_REDIRECTS = 3;
33const MANUAL_REDIRECT: RequestInit = { redirect: "manual" };
34const MEDIA_CONTENT_TYPE_PATTERN = /^(?:image|video|audio)\/[\w!#$&^.+-]+$/i;
35const ATTACHMENT_PATH_PATTERN =
36 /^\/(?:user-attachments\/assets\/[\w-]+|[^/]+\/[^/]+\/assets\/\d+\/[\w-]+)$/u;
37const LEGACY_ATTACHMENT_REPOSITORY_PATTERN = /^\/([^/]+\/[^/]+)\/assets\//u;
38const REPOSITORY_PATTERN = /^[\w.-]+\/[\w.-]+$/u;
39// GitHub's signed upload links last about five minutes.
40const SIGNED_URL_CACHE_TTL_MS = 4 * 60_000;
41const SIGNED_URL_CACHE_MAX_ENTRIES = 64;
42​
43const contentTypeOf = (response: HttpClientResponse.HttpClientResponse) =>
44 response.headers["content-type"]?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
45​
46/** The repository an upload has to be rendered in, or `undefined` when it is not an upload. */
47const uploadRepository = (request: SourceControlMediaRequest) => {
48 const target = URL.parse(request.url);
49 if (
50 target === null ||
51 target.protocol !== "https:" ||
52 target.hostname !== "github.com" ||
53 target.port !== "" ||
54 target.username !== "" ||
55 target.password !== "" ||
56 !ATTACHMENT_PATH_PATTERN.test(target.pathname)
57 ) {
58 return undefined;
59 }
60 // A legacy upload names its repository; one the caller supplies cannot override that.
61 const repository =
62 LEGACY_ATTACHMENT_REPOSITORY_PATTERN.exec(target.pathname)?.[1] ?? request.repository;
63 // An unusable explicit context must not fall back to an unrelated checkout.
64 return repository !== undefined && REPOSITORY_PATTERN.test(repository) ? repository : undefined;
65};
66​
67export const make = Effect.gen(function* () {
68 const credentials = yield* GitHubCredentials.GitHubCredentials;
69 const rest = yield* GitHubRest.GitHubRest;
70 const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
71 const signedUrls = new Map<string, { readonly at: number; readonly url: string }>();
72​
73 // Both media hosts are served by github.com's account, which is the host `gh` stores it under.
74 // Read on every fetch, so media loads as soon as the user signs in.
75 const credential = Effect.suspend(() => credentials.get("github.com")).pipe(
76 Effect.orElseSucceed(() => null),
77 );
78​
79 const fetchFollowingRedirects = Effect.fnUntraced(function* (
80 url: string,
81 headers: Readonly<Record<string, string>>,
82 token: Redacted.Redacted<string> | null,
83 ) {
84 let target = url;
85 for (let hop = 0; ; hop += 1) {
86 const authorization =
87 token !== null && isCredentialedHost(target) ? `Bearer ${Redacted.value(token)}` : null;
88 const response: HttpClientResponse.HttpClientResponse = yield* httpClient
89 .execute(
90 HttpClientRequest.get(target).pipe(
91 HttpClientRequest.setHeaders({
92 ...headers,
93 // The bytes are streamed straight through, so never let an encoding layer in.
94 "accept-encoding": "identity",
95 ...(authorization === null ? {} : { authorization }),
96 }),
97 ),
98 )
99 .pipe(
100 Effect.provideService(FetchHttpClient.RequestInit, MANUAL_REDIRECT),
101 // A signed URL's query string is a bearer capability; keep it out of tracing spans.
102 Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
103 );
104 const location = response.headers.location;
105 if (response.status < 300 || response.status >= 400) return response;
106 if (!location || hop >= MAX_REDIRECTS) return null;
107 const next = URL.parse(location, target);
108 if (next === null || next.protocol !== "https:" || next.username || next.password) {
109 return null;
110 }
111 target = next.toString();
112 }
113 });
114​
115 // An organization with SAML SSO answers a token-authenticated upload request with its sign-in
116 // page. Rendering the link as markdown in the repository's context hands back a signed URL.
117 const renderSignedUrl = Effect.fnUntraced(function* (url: string, repository: string) {
118 const rendered = yield* rest
119 .rest({
120 host: "github.com",
121 operation: "renderMarkdown",
122 method: "POST",
123 path: "markdown",
124 accept: "text/html",
125 body: { text: `![](${url})`, mode: "gfm", context: repository },
126 })
127 .pipe(Effect.orElseSucceed(() => null));
128 if (rendered === null || rendered.status !== 200 || rendered.truncated) return null;
129 for (const match of rendered.body.matchAll(/<img\b[^>]*\ssrc\s*=\s*(["'])(.*?)\1/giu)) {
130 const signed = URL.parse(match[2]?.replaceAll("&amp;", "&") ?? "");
131 if (
132 signed?.protocol === "https:" &&
133 signed.hostname === "private-user-images.githubusercontent.com" &&
134 signed.port === "" &&
135 signed.username === "" &&
136 signed.password === "" &&
137 signed.searchParams.get("jwt")
138 ) {
139 return signed.toString();
140 }
141 }
142 return null;
143 });
144​
145 return Effect.fn("GitHubMediaFetch.fetchMedia")(function* (request: SourceControlMediaRequest) {
146 const signedIn = yield* credential;
147 const repository = signedIn === null ? undefined : uploadRepository(request);
148 const cacheKey =
149 signedIn === null || repository === undefined
150 ? null
151 : [signedIn.fingerprint, repository, request.url].map(encodeURIComponent).join("|");
152 const now = yield* Clock.currentTimeMillis;
153 const cached = cacheKey === null ? undefined : signedUrls.get(cacheKey);
154 let response: HttpClientResponse.HttpClientResponse | null;
155 if (cacheKey !== null && cached !== undefined && now - cached.at < SIGNED_URL_CACHE_TTL_MS) {
156 response = yield* fetchFollowingRedirects(cached.url, request.headers, null);
157 if (response === null || response.status >= 400) signedUrls.delete(cacheKey);
158 // A signed URL can expire before its TTL; only then is a fresh one worth rendering.
159 if (response?.status !== 403) return response;
160 } else {
161 if (cacheKey !== null) signedUrls.delete(cacheKey);
162 response = yield* fetchFollowingRedirects(
163 request.url,
164 request.headers,
165 signedIn?.token ?? null,
166 );
167 if (response === null || contentTypeOf(response) !== "text/html") return response;
168 }
169 if (cacheKey === null || repository === undefined || signedIn === null) return response;
170 const signed = yield* renderSignedUrl(request.url, repository);
171 if (signed === null) return response;
172 const media = yield* fetchFollowingRedirects(signed, request.headers, null);
173 if (
174 (media?.status === 200 || media?.status === 206) &&
175 MEDIA_CONTENT_TYPE_PATTERN.test(contentTypeOf(media))
176 ) {
177 // The render can run under another account if the user switched mid-request.
178 const current = yield* credential;
179 if (current?.fingerprint === signedIn.fingerprint) {
180 if (signedUrls.size >= SIGNED_URL_CACHE_MAX_ENTRIES) {
181 const oldest = signedUrls.keys().next().value;
182 if (oldest !== undefined) signedUrls.delete(oldest);
183 }
184 signedUrls.set(cacheKey, { at: now, url: signed });
185 }
186 }
187 return media;
188 });
189});