From e305754990234619d2c870b517555d99e4717522 Mon Sep 17 00:00:00 2001 From: Steven Le Rouzic Date: Wed, 4 Jun 2025 19:27:53 +0200 Subject: Refer to ASL using archives --- .gitignore | 1 + integrity.py | 34 -------------- modules/asl/0.1.0/MODULE.bazel | 20 ++++++++ modules/asl/0.1.0/source.json | 4 ++ modules/asl/0.2.0/source.json | 7 ++- modules/asl/0.3.0/source.json | 7 ++- modules/asl/0.4.0/source.json | 7 ++- tools/add_module.py | 102 +++++++++++++++++++++++++++++++++++++++++ tools/integrity.py | 34 ++++++++++++++ 9 files changed, 170 insertions(+), 46 deletions(-) create mode 100644 .gitignore delete mode 100644 integrity.py create mode 100644 modules/asl/0.1.0/MODULE.bazel create mode 100644 modules/asl/0.1.0/source.json create mode 100644 tools/add_module.py create mode 100644 tools/integrity.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed8ebf5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file diff --git a/integrity.py b/integrity.py deleted file mode 100644 index 14bfe66..0000000 --- a/integrity.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -import hashlib -import base64 -from pathlib import Path -import requests - -def file_hash(file_path: Path) -> str: - sha256_hash = hashlib.sha256() - with open(file_path, "rb") as file: - while True: - data = file.read(65536) - if not data: - break - sha256_hash.update(data) - return "sha256-" + base64.b64encode(sha256_hash.digest()).decode() - -def url_hash(url: str) -> str: - sha256_hash = hashlib.sha256() - with requests.get(url, stream=True) as resp: - resp.raise_for_status() - while True: - data = resp.raw.read(65536) - if not data: - break - sha256_hash.update(data) - return "sha256-" + base64.b64encode(sha256_hash.digest()).decode() - -if __name__ == "__main__" and len(sys.argv) > 1: - path = Path(sys.argv[1]) - if path.exists(): - print(file_hash(path)) - else: - print(url_hash(sys.argv[1])) - diff --git a/modules/asl/0.1.0/MODULE.bazel b/modules/asl/0.1.0/MODULE.bazel new file mode 100644 index 0000000..2a71585 --- /dev/null +++ b/modules/asl/0.1.0/MODULE.bazel @@ -0,0 +1,20 @@ +# Copyright 2025 Steven Le Rouzic +# +# SPDX-License-Identifier: BSD-3-Clause + +module(name = "asl", version = "0.1.0") + +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_license", version = "1.0.0") + +bazel_dep(name = "hedron_compile_commands", dev_dependency = True) +git_override( + module_name = "hedron_compile_commands", + remote = "https://github.com/hedronvision/bazel-compile-commands-extractor.git", + commit = "4f28899228fb3ad0126897876f147ca15026151e", +) + +bazel_dep(name = "rules_python", version = "1.1.0") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.13", is_default = True) diff --git a/modules/asl/0.1.0/source.json b/modules/asl/0.1.0/source.json new file mode 100644 index 0000000..8bfb48c --- /dev/null +++ b/modules/asl/0.1.0/source.json @@ -0,0 +1,4 @@ +{ + "url": "https://bazel.stevenlr.com/blobs/asl/0.1.0-E0yyEgMYS3.tar.gz", + "integrity": "sha256-E0yyEgMYS3a8Y+oiHzdQNHvTobSXlsDjOrwvDv02Eao=" +} \ No newline at end of file diff --git a/modules/asl/0.2.0/source.json b/modules/asl/0.2.0/source.json index 1bed36d..e5f270e 100644 --- a/modules/asl/0.2.0/source.json +++ b/modules/asl/0.2.0/source.json @@ -1,5 +1,4 @@ { - "type": "git_repository", - "remote": "https://git.stevenlr.com/460nm/asl.git", - "commit": "95598e24e16ddd23a5d425e3a797f7518a93c324" -} + "url": "https://bazel.stevenlr.com/blobs/asl/0.2.0-r9c2XzBfyd.tar.gz", + "integrity": "sha256-r9c2XzBfydc55DTn/WOZm/W/BwnBNhev8adGQA0xZgs=" +} \ No newline at end of file diff --git a/modules/asl/0.3.0/source.json b/modules/asl/0.3.0/source.json index 40313be..e54c533 100644 --- a/modules/asl/0.3.0/source.json +++ b/modules/asl/0.3.0/source.json @@ -1,5 +1,4 @@ { - "type": "git_repository", - "remote": "https://git.stevenlr.com/460nm/asl.git", - "commit": "afbfd0e78176d47c495a29c795fbd3690fa0ded3" -} + "url": "https://bazel.stevenlr.com/blobs/asl/0.3.0-NvcVEwRjIK.tar.gz", + "integrity": "sha256-NvcVEwRjIKHSO2MVKO/HsemTGyoSgOzcTZTkQ6PaGHs=" +} \ No newline at end of file diff --git a/modules/asl/0.4.0/source.json b/modules/asl/0.4.0/source.json index cadfcbe..ae1fee0 100644 --- a/modules/asl/0.4.0/source.json +++ b/modules/asl/0.4.0/source.json @@ -1,5 +1,4 @@ { - "type": "git_repository", - "remote": "https://git.stevenlr.com/460nm/asl.git", - "commit": "5bca42b04941ce132426100b5f99da096b5402b6" -} + "url": "https://bazel.stevenlr.com/blobs/asl/0.4.0-BxafPMm6qK.tar.gz", + "integrity": "sha256-BxafPMm6qKvY8YLL0Zi14VNEw+FnTCeHIv86Y2xYLRI=" +} \ No newline at end of file diff --git a/tools/add_module.py b/tools/add_module.py new file mode 100644 index 0000000..c0ef016 --- /dev/null +++ b/tools/add_module.py @@ -0,0 +1,102 @@ +import sys +import subprocess +import argparse +from pathlib import Path, PurePosixPath +import tempfile +import tarfile +from integrity import file_hash +import json + +def make_git_archive(repo_path: Path, tag: str, output_path: Path): + try: + subprocess.run( + ["git", "archive", "--format=tar.gz", "--output", str(output_path.absolute()), tag], + cwd=repo_path, + check=True + ) + except subprocess.CalledProcessError as e: + print(f"Error creating git archive: {e}") + sys.exit(1) + +def extract_module_content(archive_path: Path) -> str: + with tarfile.open(archive_path, "r:gz") as tar: + file = tar.extractfile("MODULE.bazel") + if file is None: + print("Error: MODULE.bazel file not found in the archive.") + sys.exit(1) + return file.read().decode() + +def upload_to_blob_storage(local_path: Path, remote_path: str, scp_port: int = 22): + try: + ssh_login = remote_path.split(":")[0] + ssh_path = str(PurePosixPath(remote_path.split(":")[1]).parent) + + subprocess.run( + ["ssh", "-p", str(scp_port), ssh_login, f"mkdir -p {ssh_path}"], + check=True, + stdout=sys.stdout, + stderr=sys.stderr, + ) + + subprocess.run( + ["scp", "-P", str(scp_port), str(local_path), remote_path], + check=True, + stdout=sys.stdout, + stderr=sys.stderr, + ) + except subprocess.CalledProcessError as e: + print(f"Error uploading to blob storage: {e}") + sys.exit(1) + +MODULES_DIR = Path(__file__).parent.parent / "modules" + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Add a module to the registry.") + parser.add_argument("module_name", type=str, help="Name of the module to add") + parser.add_argument("module_version", type=str, help="Version of the module to add") + parser.add_argument("local_git_repo_path", type=str, help="Path to the local git repository") + parser.add_argument("remote_blob_path", type=str, help="SCP-compatible path to the remote blob storage folder") + parser.add_argument("-t", "--tag", type=str, default=None, help="Git tag for the module version") + parser.add_argument("-p", "--scp_port", type=int, default=22, help="Port for SCP connection (default: 22)") + + args = parser.parse_args() + + module_name = args.module_name + module_version = args.module_version + local_git_repo_path = Path(args.local_git_repo_path) + git_repo_tag = args.tag or f"v{module_version}" + + if not local_git_repo_path.exists(): + print(f"Error: The specified local git repository path '{local_git_repo_path}' does not exist.") + sys.exit(1) + + registry_module_path = MODULES_DIR / module_name / module_version + if registry_module_path.exists(): + print(f"Error: The module '{module_name}' version '{module_version}' already exists in the registry.") + sys.exit(1) + + tmp_dir = tempfile.TemporaryDirectory() + + archive_file = Path(tmp_dir.name) / f"archive.tar.gz" + make_git_archive(local_git_repo_path, git_repo_tag, archive_file) + + module_content = extract_module_content(archive_file) + integrity = file_hash(archive_file) + partial_hash = integrity[7:17] + + registry_module_path.mkdir(parents=True, exist_ok=True) + with open(registry_module_path / "MODULE.bazel", "wb+") as f: + f.write(module_content.encode()) + + blob_path = f"{module_name}/{module_version}-{partial_hash}.tar.gz" + remote_blob_path = f"{args.remote_blob_path}/{blob_path}" + + upload_to_blob_storage(archive_file, remote_blob_path, args.scp_port) + + source_json = { + "url": f"https://bazel.stevenlr.com/blobs/{blob_path}", + "integrity": integrity, + } + + with open(registry_module_path / "source.json", "wb+") as f: + f.write(json.dumps(source_json, indent=2).encode()) diff --git a/tools/integrity.py b/tools/integrity.py new file mode 100644 index 0000000..e1b2572 --- /dev/null +++ b/tools/integrity.py @@ -0,0 +1,34 @@ +import sys +import hashlib +import base64 +from pathlib import Path +import requests + +def file_hash(file_path: Path) -> str: + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as file: + while True: + data = file.read(65536) + if not data: + break + sha256_hash.update(data) + return "sha256-" + base64.b64encode(sha256_hash.digest()).decode() + +def url_hash(url: str) -> str: + sha256_hash = hashlib.sha256() + with requests.get(url, stream=True) as resp: + resp.raise_for_status() + while True: + data = resp.raw.read(65536) + if not data: + break + sha256_hash.update(data) + return "sha256-" + base64.b64encode(sha256_hash.digest()).decode() + +if __name__ == "__main__" and len(sys.argv) > 1: + path = Path(sys.argv[1]) + if path.exists(): + print(file_hash(path)) + else: + print(url_hash(sys.argv[1])) + -- cgit