Support RSA keys and configure external ChuSQL submodules

scripts/init-deps.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Initialize pinned external submodules, preferring local repositories over SSH."""
+import argparse
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[1]
+PROJECTS = ['ChuSQL', 'ChuSQL-RustDriver']
+PREFIX = 'ChuGit:/srv/git/'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--database-repo', type=Path, help='local clone/bare repository of ChuSQL')
+    parser.add_argument('--driver-repo', type=Path, help='local clone/bare repository of ChuSQL-RustDriver')
+    args = parser.parse_args()
+    git_args = []
+    local_config = ROOT / 'config/git-local.config'
+    if not args.database_repo and not args.driver_repo and all((Path('/srv/git') / f'{name}.git').is_dir() for name in PROJECTS):
+        current = subprocess.run(['git', 'config', '--local', '--get-all', 'include.path'], cwd=ROOT, text=True, capture_output=True, check=False)
+        if str(local_config) not in current.stdout.splitlines():
+            subprocess.run(['git', 'config', '--local', '--add', 'include.path', str(local_config)], cwd=ROOT, check=True)
+        git_args.extend(['-c', f'include.path={local_config}'])
+    for name, override in zip(PROJECTS, [args.database_repo, args.driver_repo]):
+        source = override or Path('/srv/git') / f'{name}.git'
+        if override and not source.is_dir():
+            parser.error(f'Local repository does not exist: {source}')
+        if source.is_dir():
+            source = source.resolve()
+            subprocess.run(['git', '-C', str(source), 'rev-parse', '--git-dir'], check=True, capture_output=True)
+            key = f'url.{source.as_uri()}.insteadOf'
+            url = f'{PREFIX}{name}.git'
+            subprocess.run(['git', 'config', '--local', '--replace-all', key, url], cwd=ROOT, check=True)
+            git_args.extend(['-c', f'{key}={url}'])
+            print(f'{name}: using local repository {source}', flush=True)
+    # The file protocol is enabled only for this explicit, fixed set of dependencies.
+    if git_args:
+        git_args.extend(['-c', 'protocol.file.allow=always'])
+    subprocess.run(['git', *git_args, 'submodule', 'update', '--init', '--', *[f'external/{name}' for name in PROJECTS]], cwd=ROOT, check=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/prepare-deploy.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Build the API and copy existing local ChuSQL release artifacts into the Docker context."""
+"""Build the API and stage external ChuSQL artifacts without committing dependencies."""
 import argparse
 from pathlib import Path
 import shutil
@@ -8,17 +8,27 @@ import subprocess
 root = Path(__file__).resolve().parents[1]
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument('--no-build', action='store_true', help='use an existing API release binary')
+parser.add_argument('--chusql-source', type=Path, default=root / 'external/ChuSQL', help='initialized/built external ChuSQL checkout')
+parser.add_argument('--build-database', action='store_true', help='build external database artifacts before staging')
 args = parser.parse_args()
+chusql = args.chusql_source.resolve()
+if not (root / 'external/ChuSQL-RustDriver/Cargo.toml').is_file():
+    parser.error('Initialize external dependencies with scripts/init-deps.py first.')
+if not (chusql / 'chusql-core/storage/Cargo.toml').is_file():
+    parser.error(f'ChuSQL source is missing: {chusql}; initialize dependencies first.')
 if not args.no_build:
     subprocess.run(['cargo', 'build', '--locked', '--release'], cwd=root, check=True)
-out = root / 'deploy' / 'bin'
+if args.build_database:
+    subprocess.run(['cargo', 'build', '--locked', '--release'], cwd=chusql / 'chusql-core/storage', check=True)
+    for package in ['chusql-server', 'chusql-bootstrap']:
+        subprocess.run(['stack', 'build', '--fast'], cwd=chusql / package, check=True)
+out = root / 'deploy/bin'
 out.mkdir(parents=True, exist_ok=True)
 shutil.copy2(root / 'target/release/chublog-api', out / 'chublog-api')
-chusql = root.parent / 'ChuSQL'
 for package, executable in [('chusql-server', 'chusql-server'), ('chusql-bootstrap', 'csql-bootstrap')]:
     matches = list((chusql / package / '.stack-work/install').glob(f'*/**/bin/{executable}'))
     if len(matches) != 1:
-        raise SystemExit(f'Expected one built {executable}, found {len(matches)}; build {package} first using its stack.yaml.')
+        raise SystemExit(f'Expected one built {executable}, found {len(matches)}; use --build-database or select a built --chusql-source.')
     shutil.copy2(matches[0], out / executable)
 shutil.copy2(chusql / 'chusql-core/storage/target/release/libchusql_core_storage.so', out)
-print('Prepared deploy/bin. Build and run Compose from this same Linux architecture (Debian 13).')
+print('Prepared ignored deploy/bin. Build and run Compose on the same Debian 13 architecture.')

scripts/smoke.py

@@ -3,6 +3,7 @@
 Uses an isolated Compose project and removes only that project's containers/volumes.
 Run prepare-deploy.py first. Ports 18080, 18443, 12222 must be free.
 """
+import argparse
 import http.cookiejar
 import json
 import os
@@ -16,6 +17,9 @@ import urllib.request
 
 ROOT = Path(__file__).resolve().parents[1]
 ORIGIN = 'http://localhost:18080'
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('--key-type', choices=['ed25519', 'rsa'], default='ed25519')
+args = parser.parse_args()
 
 
 def run(args, **kwargs):
@@ -77,7 +81,7 @@ with tempfile.TemporaryDirectory(prefix='chublog-smoke-') as temporary:
         publisher = Client()
         publisher.call('POST', '/api/auth/register', {'username': 'alice', 'password': password})
         publisher.csrf = publisher.call('POST', '/api/auth/login', {'username': 'alice', 'password': password})['csrf']
-        run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(tmp / 'key')])
+        run(['ssh-keygen', '-q', '-t', args.key_type, '-b', '3072', '-N', '', '-f', str(tmp / 'key')])
         publisher.call('PUT', '/api/account/ssh-key', {'public_key': (tmp / 'key.pub').read_text().strip()})
         publisher.call('GET', '/api/admin/users', expected=403)
         env = os.environ.copy()

scripts/test-init-deps.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Verify pinned submodules initialize from local repositories without an SSH connection."""
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+SCRIPT = Path(__file__).with_name('init-deps.py')
+
+
+class InitializeDependencies(unittest.TestCase):
+    def test_local_rewrite_fetches_only_pinned_dependency_commits(self):
+        with tempfile.TemporaryDirectory(prefix='chublog deps ') as temporary:
+            root = Path(temporary)
+            project = root / 'blog'
+            project.mkdir()
+            (project / 'scripts').mkdir()
+            shutil.copy2(SCRIPT, project / 'scripts/init-deps.py')
+            env = {**os.environ, 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_CONFIG_NOSYSTEM': '1'}
+
+            def git(cwd, *args):
+                return subprocess.run(['git', *args], cwd=cwd, env=env, check=True, text=True, capture_output=True).stdout.strip()
+
+            git(project, 'init', '--initial-branch=main')
+            modules = []
+            sources = []
+            for name in ['ChuSQL', 'ChuSQL-RustDriver']:
+                source = root / name
+                source.mkdir()
+                git(source, 'init', '--initial-branch=main')
+                (source / 'source.txt').write_text(f'{name} committed source')
+                git(source, 'add', '.')
+                git(source, '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'Dependency version')
+                oid = git(source, 'rev-parse', 'HEAD')
+                # Dirty source must never leak into the blog or fetched dependency.
+                (source / 'source.txt').write_text('uncommitted changes')
+                path = f'external/{name}'
+                modules.append(f'[submodule "{path}"]\n path = {path}\n url = ChuGit:/srv/git/{name}.git\n')
+                git(project, 'update-index', '--add', '--cacheinfo', f'160000,{oid},{path}')
+                sources.append(str(source))
+            (project / '.gitmodules').write_text(''.join(modules))
+            git(project, 'add', '.gitmodules')
+            subprocess.run(['python3', 'scripts/init-deps.py', '--database-repo', sources[0], '--driver-repo', sources[1]], cwd=project, env=env, check=True, capture_output=True)
+            for name in ['ChuSQL', 'ChuSQL-RustDriver']:
+                self.assertEqual((project / f'external/{name}/source.txt').read_text(), f'{name} committed source')
+            self.assertTrue(all(line.startswith('160000') for line in git(project, 'ls-files', '--stage', 'external').splitlines()))
+            # Running the initialization a second time must preserve the same pins.
+            subprocess.run(['python3', 'scripts/init-deps.py', '--database-repo', sources[0], '--driver-repo', sources[1]], cwd=project, env=env, check=True, capture_output=True)
+
+
+if __name__ == '__main__':
+    unittest.main()