chore: preserve paused QBit work and document remaining issues
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Failing after 7s
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Failing after 6s
QRun-IO/ci-policy: native / native-result (push) Successful in 6s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Failing after 7s
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Failing after 6s
QRun-IO/ci-policy: native / native-result (push) Successful in 6s
Save the current implementation and README, status, and known-issue documentation at the user-requested stop. No completed release or deployment is claimed. Co-Authored-By: Claude <noreply@anthropic.com> Co-Authored-By: GPT-6 Codex <noreply@openai.com>
This commit is contained in:
257 files changed
+47918
No files matched your search
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package com.kingsrook.qbits.scripting.github;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.Flow;
|
||||
|
||||
/*******************************************************************************
|
||||
** Enforce body size while HTTP bytes arrive, before JSON allocation.
|
||||
*******************************************************************************/
|
||||
final class BoundedBody implements HttpResponse.BodySubscriber<byte[]>
|
||||
{
|
||||
private final int maximum;
|
||||
private final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
private final CompletableFuture<byte[]> result = new CompletableFuture<>();
|
||||
private Flow.Subscription subscription;
|
||||
|
||||
/***************************************************************************
|
||||
** Per-request maximum includes encoded JSON overhead.
|
||||
***************************************************************************/
|
||||
BoundedBody(int maximum)
|
||||
{
|
||||
this.maximum = maximum;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** The HTTP future completes only after bounded body delivery.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public CompletionStage<byte[]> getBody()
|
||||
{
|
||||
return (result);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Pull one chunk at a time to bound outstanding input.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void onSubscribe(Flow.Subscription value)
|
||||
{
|
||||
subscription = value;
|
||||
value.request(1);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Reject oversized content without retaining the remaining body.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void onNext(List<ByteBuffer> chunks)
|
||||
{
|
||||
for(ByteBuffer chunk : chunks)
|
||||
{
|
||||
if(chunk.remaining() > maximum - bytes.size())
|
||||
{
|
||||
subscription.cancel();
|
||||
result.completeExceptionally(new IllegalArgumentException("GitHub response exceeds limit"));
|
||||
return;
|
||||
}
|
||||
byte[] value = new byte[chunk.remaining()];
|
||||
chunk.get(value);
|
||||
bytes.writeBytes(value);
|
||||
}
|
||||
subscription.request(1);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Propagate transport failure without logging its details.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void onError(Throwable error)
|
||||
{
|
||||
result.completeExceptionally(error);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Final immutable response bytes.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void onComplete()
|
||||
{
|
||||
result.complete(bytes.toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package com.kingsrook.qbits.scripting.github;
|
||||
|
||||
import com.kingsrook.qbits.scripting.ScriptingException;
|
||||
import com.kingsrook.qbits.scripting.service.Actor;
|
||||
|
||||
/*******************************************************************************
|
||||
** Host-managed repository authorization and credential lookup. Implementations
|
||||
** authorize the current actor/project/repository on EVERY call; deny by throwing.
|
||||
** Return null only for explicitly authorized anonymous public access. Tokens
|
||||
** stay in trusted HTTP requests, never source records, contexts or diagnostics.
|
||||
*******************************************************************************/
|
||||
@FunctionalInterface
|
||||
public interface GitHubAccess
|
||||
{
|
||||
/***************************************************************************
|
||||
** Resolve a short-lived read-only credential from the host's existing store.
|
||||
***************************************************************************/
|
||||
String tokenFor(Actor actor, Integer scriptId, String repository) throws ScriptingException;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package com.kingsrook.qbits.scripting.github;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.CodingErrorAction;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qbits.scripting.ScriptingException;
|
||||
import com.kingsrook.qbits.scripting.service.Actor;
|
||||
import com.kingsrook.qbits.scripting.source.GitCommitRef;
|
||||
import com.kingsrook.qbits.scripting.source.GitSourceImporter;
|
||||
import com.kingsrook.qbits.scripting.source.SourceFile;
|
||||
import com.kingsrook.qbits.scripting.source.SourceSnapshot;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
/*******************************************************************************
|
||||
** Explicit GitHub.com exact-commit import. No clone, archives, redirects,
|
||||
** moving refs, URLs from responses, package installation or repository hooks.
|
||||
** Selected subtree must contain only ordinary UTF-8 .ts files and directories.
|
||||
*******************************************************************************/
|
||||
public final class GitHubSourceImporter implements GitSourceImporter
|
||||
{
|
||||
private static final int MAX_RESPONSE = 1024 * 1024;
|
||||
private static final int MAX_TREE_ENTRIES = 1000;
|
||||
private final GitHubAccess access;
|
||||
private final Reader reader;
|
||||
|
||||
/***************************************************************************
|
||||
** Trusted fixture seam; production requests always use the fixed GitHub host.
|
||||
***************************************************************************/
|
||||
@FunctionalInterface
|
||||
interface Reader
|
||||
{
|
||||
/***********************************************************************
|
||||
** Read one bounded response within the remaining operation deadline.
|
||||
***********************************************************************/
|
||||
byte[] get(String path, String token, int maximum, Duration timeout) throws Exception;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Host must explicitly install its existing repository authorization policy.
|
||||
***************************************************************************/
|
||||
public GitHubSourceImporter(GitHubAccess access)
|
||||
{
|
||||
this(access, GitHubSourceImporter::read);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Package-private deterministic HTTP-response fixtures, never a browser input.
|
||||
***************************************************************************/
|
||||
GitHubSourceImporter(GitHubAccess access, Reader reader)
|
||||
{
|
||||
this.access = Objects.requireNonNull(access);
|
||||
this.reader = Objects.requireNonNull(reader);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Verify exact commit identity, selected tree, blob size/hash and source policy.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public SourceSnapshot importSnapshot(Actor actor, Integer scriptId, GitCommitRef ref) throws ScriptingException
|
||||
{
|
||||
Actor.requireCurrentIdentity(actor.id());
|
||||
long deadline = System.nanoTime() + Duration.ofSeconds(45).toNanos();
|
||||
String base = "/repos/" + ref.repository() + "/git/";
|
||||
try
|
||||
{
|
||||
JSONObject commit = get(actor, scriptId, ref, base + "commits/" + ref.commitSha(), deadline);
|
||||
require(ref.commitSha().equals(commit.getString("sha")), "Commit identity mismatch");
|
||||
String tree = sha(commit.getJSONObject("tree").getString("sha"));
|
||||
if(ref.subdirectory() != null)
|
||||
{
|
||||
for(String segment : ref.subdirectory().split("/"))
|
||||
{
|
||||
JSONArray entries = tree(actor, scriptId, ref, base, tree, false, deadline);
|
||||
String next = null;
|
||||
for(int i = 0; i < entries.length(); i++)
|
||||
{
|
||||
JSONObject entry = entries.getJSONObject(i);
|
||||
if(segment.equals(entry.getString("path")))
|
||||
{
|
||||
require(next == null && "tree".equals(entry.getString("type")) && "040000".equals(entry.getString("mode")), "Subdirectory must be an ordinary tree");
|
||||
next = sha(entry.getString("sha"));
|
||||
}
|
||||
}
|
||||
require(next != null, "Selected source directory is missing");
|
||||
tree = next;
|
||||
}
|
||||
}
|
||||
JSONArray entries = tree(actor, scriptId, ref, base, tree, true, deadline);
|
||||
List<JSONObject> blobs = new ArrayList<>();
|
||||
Set<String> paths = new HashSet<>();
|
||||
long total = 0;
|
||||
for(int i = 0; i < entries.length(); i++)
|
||||
{
|
||||
JSONObject entry = entries.getJSONObject(i);
|
||||
String path = entry.getString("path");
|
||||
require(paths.add(path.toLowerCase(java.util.Locale.ROOT)), "Duplicate source tree path");
|
||||
String type = entry.getString("type");
|
||||
String mode = entry.getString("mode");
|
||||
sha(entry.getString("sha"));
|
||||
if("tree".equals(type) && "040000".equals(mode))
|
||||
{
|
||||
SourceFile.validatePath(path + "/placeholder.ts");
|
||||
continue;
|
||||
}
|
||||
require("blob".equals(type) && "100644".equals(mode), "Only ordinary non-executable source files are accepted");
|
||||
SourceFile.validatePath(path);
|
||||
long size = size(entry);
|
||||
require(size <= SourceFile.MAX_CONTENT_BYTES, "Source file exceeds limit");
|
||||
total += size;
|
||||
require(total <= SourceSnapshot.MAX_TOTAL_BYTES && blobs.size() < SourceSnapshot.MAX_FILES, "Source snapshot exceeds limit");
|
||||
blobs.add(entry);
|
||||
}
|
||||
List<SourceFile> files = new ArrayList<>();
|
||||
for(JSONObject entry : blobs)
|
||||
{
|
||||
String expected = entry.getString("sha");
|
||||
JSONObject blob = get(actor, scriptId, ref, base + "blobs/" + expected, deadline);
|
||||
require(expected.equals(blob.getString("sha")) && "base64".equals(blob.getString("encoding")), "Invalid blob identity or encoding");
|
||||
byte[] bytes = Base64.getDecoder().decode(blob.getString("content").replace("\n", "").replace("\r", ""));
|
||||
require(bytes.length == size(entry) && bytes.length == size(blob), "Blob length mismatch");
|
||||
MessageDigest hash = MessageDigest.getInstance("SHA-1");
|
||||
hash.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII));
|
||||
require(expected.equals(SourceSnapshot.hex(hash.digest(bytes))), "Blob content hash mismatch");
|
||||
files.add(new SourceFile(entry.getString("path"), utf8(bytes)));
|
||||
}
|
||||
return (SourceSnapshot.of(ref.origin(), files));
|
||||
}
|
||||
catch(ScriptingException error)
|
||||
{
|
||||
throw error;
|
||||
}
|
||||
catch(Exception error)
|
||||
{
|
||||
throw new ScriptingException(ScriptingException.Kind.INVALID, "Repository response violates the source import policy");
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Reject partial trees; truncation must never silently drop source files.
|
||||
***************************************************************************/
|
||||
private JSONArray tree(Actor actor, Integer project, GitCommitRef ref, String base, String sha, boolean recursive, long deadline) throws ScriptingException
|
||||
{
|
||||
JSONObject response = get(actor, project, ref, base + "trees/" + sha + (recursive ? "?recursive=1" : ""), deadline);
|
||||
require(sha.equals(response.getString("sha")) && Boolean.FALSE.equals(response.get("truncated")), "Incomplete or mismatched source tree");
|
||||
JSONArray entries = response.getJSONArray("tree");
|
||||
require(entries.length() <= MAX_TREE_ENTRIES, "Tree entry count exceeds limit");
|
||||
return (entries);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Each request reauthorizes current actor/repository; no response URL is used.
|
||||
***************************************************************************/
|
||||
private JSONObject get(Actor actor, Integer project, GitCommitRef ref, String path, long deadline) throws ScriptingException
|
||||
{
|
||||
Actor.requireCurrentIdentity(actor.id());
|
||||
String token = access.tokenFor(actor, project, ref.repository());
|
||||
long remaining = deadline - System.nanoTime();
|
||||
if(remaining <= 0)
|
||||
{
|
||||
throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Source import deadline exceeded");
|
||||
}
|
||||
try
|
||||
{
|
||||
byte[] bytes = reader.get(path, token, MAX_RESPONSE, Duration.ofNanos(Math.min(remaining, Duration.ofSeconds(15).toNanos())));
|
||||
require(bytes.length <= MAX_RESPONSE, "Repository response exceeds limit");
|
||||
return (new JSONObject(utf8(bytes)));
|
||||
}
|
||||
catch(ScriptingException error)
|
||||
{
|
||||
throw error;
|
||||
}
|
||||
catch(Exception error)
|
||||
{
|
||||
if(error instanceof InterruptedException)
|
||||
{
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Repository content is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Fixed public API host; managed credentials cannot follow redirects elsewhere.
|
||||
***************************************************************************/
|
||||
private static byte[] read(String path, String token, int maximum, Duration timeout) throws Exception
|
||||
{
|
||||
try(HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).connectTimeout(Duration.ofSeconds(5)).build())
|
||||
{
|
||||
var request = HttpRequest.newBuilder(URI.create("https://api.github.com" + path)).timeout(timeout)
|
||||
.header("Accept", "application/vnd.github+json").header("X-GitHub-Api-Version", "2026-03-10")
|
||||
.header("User-Agent", "QQQ-Scripting").GET();
|
||||
if(token != null)
|
||||
{
|
||||
request.header("Authorization", "Bearer " + token);
|
||||
}
|
||||
var future = client.sendAsync(request.build(), info -> new BoundedBody(maximum));
|
||||
try
|
||||
{
|
||||
var response = future.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
|
||||
require(response.statusCode() == 200, "Repository request refused");
|
||||
return (response.body());
|
||||
}
|
||||
finally
|
||||
{
|
||||
future.cancel(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Numeric byte counts cannot be coerced from strings or fractional JSON.
|
||||
***************************************************************************/
|
||||
private static long size(JSONObject value) throws ScriptingException
|
||||
{
|
||||
Object size = value.get("size");
|
||||
require(size instanceof Integer || size instanceof Long, "Invalid blob size");
|
||||
long result = ((Number) size).longValue();
|
||||
require(result >= 0 && result <= SourceFile.MAX_CONTENT_BYTES, "Blob size exceeds limit");
|
||||
return (result);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** API object names are full immutable SHA-1 identifiers, never paths or refs.
|
||||
***************************************************************************/
|
||||
private static String sha(String value) throws ScriptingException
|
||||
{
|
||||
require(value.matches("[0-9a-f]{40}"), "Invalid Git object identity");
|
||||
return (value);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Reject malformed UTF-8 instead of changing source bytes during decoding.
|
||||
***************************************************************************/
|
||||
private static String utf8(byte[] bytes) throws Exception
|
||||
{
|
||||
return (StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString());
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Fail the complete import; never return a filtered partial source snapshot.
|
||||
***************************************************************************/
|
||||
private static void require(boolean condition, String message) throws ScriptingException
|
||||
{
|
||||
if(!condition)
|
||||
{
|
||||
throw new ScriptingException(ScriptingException.Kind.INVALID, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package com.kingsrook.qbits.scripting.github;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.Base64;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Flow;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qbits.scripting.ScriptingException;
|
||||
import com.kingsrook.qbits.scripting.service.Actor;
|
||||
import com.kingsrook.qbits.scripting.source.GitCommitRef;
|
||||
import com.kingsrook.qbits.scripting.source.SourceSnapshot;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QUser;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/*******************************************************************************
|
||||
** Trusted GitHub HTTP response fixtures. No repository is fetched or executed.
|
||||
*******************************************************************************/
|
||||
class GitHubSourceImporterTest
|
||||
{
|
||||
/***************************************************************************
|
||||
** Establish an identified local fixture session without repository access.
|
||||
***************************************************************************/
|
||||
@BeforeEach
|
||||
void setup()
|
||||
{
|
||||
QContext.init(null, new QSession().withUser(new QUser().withIdReference("author")));
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Drop fixture identity after each test.
|
||||
***************************************************************************/
|
||||
@AfterEach
|
||||
void cleanup()
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Actor is always derived from the actual current fixture session.
|
||||
***************************************************************************/
|
||||
private Actor everything() throws ScriptingException
|
||||
{
|
||||
return (Actor.fromSession(QContext.getQSession()));
|
||||
}
|
||||
|
||||
private static final String COMMIT = "a".repeat(40);
|
||||
private static final String TREE = "b".repeat(40);
|
||||
private static final GitCommitRef REF = new GitCommitRef("acme/scripts", COMMIT, null);
|
||||
|
||||
/***************************************************************************
|
||||
** Exact commit/tree/blob provenance, canonical files and per-request access.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void importsPinnedSourceWithCurrentRepositoryAuthorization() throws Exception
|
||||
{
|
||||
Map<String, JSONObject> responses = fixture("export const value = 1;");
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
var importer = importer(responses, calls);
|
||||
var first = importer.importSnapshot(everything(), 42, REF);
|
||||
var second = importer.importSnapshot(everything(), 42, REF);
|
||||
assertEquals(first.sha256(), second.sha256());
|
||||
assertEquals(REF.origin(), first.origin());
|
||||
assertEquals("src/main.ts", first.files().getFirst().path());
|
||||
assertTrue(first.verify());
|
||||
assertEquals(6, calls.get());
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Unknown files, executable files, symlinks and submodules are all refused.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void refusesUnsafeTreeEntriesBeforeFetchingAnyBlob() throws Exception
|
||||
{
|
||||
for(String mode : List.of("100755", "120000", "160000"))
|
||||
{
|
||||
var fixture = fixture("source");
|
||||
fixture.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("mode", mode);
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
assertThrows(ScriptingException.class, () -> importer(fixture, calls).importSnapshot(everything(), 42, REF));
|
||||
assertEquals(2, calls.get());
|
||||
}
|
||||
for(String path : List.of("package.json", "../main.ts", "/main.ts", "src\\main.ts", "file.zip"))
|
||||
{
|
||||
var fixture = fixture("source");
|
||||
fixture.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("path", path);
|
||||
assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, REF));
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Missing immutable identity or partial trees cannot become usable snapshots.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void rejectsTruncationCommitMismatchAndOversize() throws Exception
|
||||
{
|
||||
var truncated = fixture("source");
|
||||
truncated.get("trees/" + TREE + "?recursive=1").put("truncated", true);
|
||||
assertThrows(ScriptingException.class, () -> importer(truncated, new AtomicInteger()).importSnapshot(everything(), 42, REF));
|
||||
var mismatch = fixture("source");
|
||||
mismatch.get("commits/" + COMMIT).put("sha", TREE);
|
||||
assertThrows(ScriptingException.class, () -> importer(mismatch, new AtomicInteger()).importSnapshot(everything(), 42, REF));
|
||||
var oversized = fixture("source");
|
||||
oversized.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("size", 300000);
|
||||
assertThrows(ScriptingException.class, () -> importer(oversized, new AtomicInteger()).importSnapshot(everything(), 42, REF));
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Bytes must match their Git blob hash, declared size and strict UTF-8.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void rejectsChangedBlobAndEncoding() throws Exception
|
||||
{
|
||||
for(String field : List.of("content", "size", "encoding", "sha"))
|
||||
{
|
||||
var fixture = fixture("source");
|
||||
var blob = fixture.entrySet().stream().filter(entry -> entry.getKey().startsWith("blobs/")).findFirst().orElseThrow().getValue();
|
||||
blob.put(field, switch(field)
|
||||
{
|
||||
case "content" -> Base64.getEncoder().encodeToString("change".getBytes(StandardCharsets.UTF_8));
|
||||
case "size" -> 1;
|
||||
case "encoding" -> "utf-8";
|
||||
default -> COMMIT;
|
||||
});
|
||||
assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, REF));
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Managed repository denial prevents all HTTP traffic and cannot expose tokens.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void deniedAccessDoesNotFetch() throws Exception
|
||||
{
|
||||
var importer = new GitHubSourceImporter((actor, project, repository) ->
|
||||
{
|
||||
throw new ScriptingException(ScriptingException.Kind.DENIED, "Repository is not authorized");
|
||||
}, (path, token, maximum, timeout) ->
|
||||
{
|
||||
throw new AssertionError("Denied actor must not fetch");
|
||||
});
|
||||
assertThrows(ScriptingException.class, () -> importer.importSnapshot(everything(), 42, REF));
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Select an exact ordinary subtree; siblings are never downloaded or imported.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void selectsPinnedSubtreeAndRejectsDirectorySymlink() throws Exception
|
||||
{
|
||||
var fixture = fixture("source");
|
||||
String child = "c".repeat(40);
|
||||
var original = fixture.remove("trees/" + TREE + "?recursive=1").put("sha", child);
|
||||
fixture.put("trees/" + child + "?recursive=1", original);
|
||||
fixture.put("trees/" + TREE, new JSONObject().put("sha", TREE).put("truncated", false).put("tree", List.of(
|
||||
Map.of("path", "scripts", "type", "tree", "mode", "040000", "sha", child),
|
||||
Map.of("path", "README.md", "type", "blob", "mode", "100644", "sha", COMMIT))));
|
||||
var ref = new GitCommitRef("acme/scripts", COMMIT, "scripts");
|
||||
assertEquals(ref.origin(), importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, ref).origin());
|
||||
fixture.get("trees/" + TREE).getJSONArray("tree").getJSONObject(0).put("mode", "120000");
|
||||
assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, ref));
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Transport bounds apply while streaming, rather than after unbounded allocation.
|
||||
***************************************************************************/
|
||||
@Test
|
||||
void boundedBodyCancelsOversizedStream()
|
||||
{
|
||||
var body = new BoundedBody(2);
|
||||
AtomicInteger canceled = new AtomicInteger();
|
||||
body.onSubscribe(new Flow.Subscription()
|
||||
{
|
||||
/*******************************************************************
|
||||
** Fixture supplies chunks explicitly.
|
||||
*******************************************************************/
|
||||
@Override
|
||||
public void request(long count) {}
|
||||
|
||||
/*******************************************************************
|
||||
** Observe immediate cancellation.
|
||||
*******************************************************************/
|
||||
@Override
|
||||
public void cancel()
|
||||
{
|
||||
canceled.incrementAndGet();
|
||||
}
|
||||
});
|
||||
body.onNext(List.of(ByteBuffer.wrap(new byte[3])));
|
||||
assertTrue(body.getBody().toCompletableFuture().isCompletedExceptionally());
|
||||
assertEquals(1, canceled.get());
|
||||
var small = new BoundedBody(4);
|
||||
small.onSubscribe(new Flow.Subscription()
|
||||
{
|
||||
/*******************************************************************
|
||||
** Fixture supplies chunks explicitly.
|
||||
*******************************************************************/
|
||||
@Override
|
||||
public void request(long count) {}
|
||||
|
||||
/*******************************************************************
|
||||
** Successful fixture does not need cancellation.
|
||||
*******************************************************************/
|
||||
@Override
|
||||
public void cancel() {}
|
||||
});
|
||||
small.onNext(List.of(ByteBuffer.wrap(new byte[2])));
|
||||
small.onComplete();
|
||||
assertFalse(small.getBody().toCompletableFuture().isCompletedExceptionally());
|
||||
assertEquals(2, small.getBody().toCompletableFuture().join().length);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Fixture object responses use the actual commit/tree/blob API shapes.
|
||||
***************************************************************************/
|
||||
private Map<String, JSONObject> fixture(String content) throws Exception
|
||||
{
|
||||
byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
|
||||
MessageDigest hash = MessageDigest.getInstance("SHA-1");
|
||||
hash.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII));
|
||||
String blob = SourceSnapshot.hex(hash.digest(bytes));
|
||||
Map<String, JSONObject> result = new HashMap<>();
|
||||
result.put("commits/" + COMMIT, new JSONObject().put("sha", COMMIT).put("tree", Map.of("sha", TREE)));
|
||||
result.put("trees/" + TREE + "?recursive=1", new JSONObject().put("sha", TREE).put("truncated", false).put("tree",
|
||||
List.of(Map.of("path", "src/main.ts", "mode", "100644", "type", "blob", "sha", blob, "size", bytes.length))));
|
||||
result.put("blobs/" + blob, new JSONObject().put("sha", blob).put("encoding", "base64").put("size", bytes.length)
|
||||
.put("content", Base64.getEncoder().encodeToString(bytes)));
|
||||
return (result);
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
** Assert the trusted credential stays only at the HTTP boundary.
|
||||
***************************************************************************/
|
||||
private GitHubSourceImporter importer(Map<String, JSONObject> responses, AtomicInteger calls)
|
||||
{
|
||||
return (new GitHubSourceImporter((actor, project, repository) ->
|
||||
{
|
||||
assertEquals(42, project);
|
||||
assertEquals(REF.repository(), repository);
|
||||
calls.incrementAndGet();
|
||||
return ("fixture-credential");
|
||||
}, (path, token, maximum, timeout) ->
|
||||
{
|
||||
assertEquals("fixture-credential", token);
|
||||
assertTrue(timeout.toSeconds() <= 15);
|
||||
return (responses.get(path.substring(("/repos/" + REF.repository() + "/git/").length())).toString().getBytes(StandardCharsets.UTF_8));
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user