Merge remote-tracking branch 'upstream/main'

.gitlab-ci.yml

@@ -147,60 +147,37 @@ release_job:
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
-create_build_machines:
-  stage: build_packages
+create_build_machine:
+  stage: prepare
   tags:
     - build-orchestration
   script:
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-deb
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create arm64-deb
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh create amd64-rpm
+    - uv --directory scripts/cicd-python sync
+    - uv --directory scripts/cicd-python run veilid_release_utils.py --create-build-machine
+  artifacts:
+    paths: 
+      - scripts/cicd-python/config.json
+    expire_in: 6 hours
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
-package_amd64_deb:
+package_linux:
   stage: build_packages
-  needs:
-    - create_build_machines
   tags:
     - build-amd64-deb
   script:
     - earthly bootstrap
-    - earthly +package-linux-amd64-deb
-    - bash scripts/cicd/build-machine/scp-amd64-debs-to-orchestrator.sh
-  rules:
-    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
-
-package_arm64_deb:
-  stage: build_packages
-  needs:
-    - create_build_machines
-  tags:
-    - build-arm64-deb
-  script:
-    - earthly bootstrap
-    - earthly +package-linux-arm64-deb
-    - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
-  rules:
-    - if: '$CI_COMMIT_TAG =~ /v\d.+/'
-
-package_amd64_rpm:
-  stage: build_packages
-  needs:
-    - create_build_machines
-  tags:
-    - build-amd64-rpm
-  script:
-    - earthly bootstrap
-    - earthly +package-linux-amd64-rpm
-    - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
+    - earthly +package-linux
+  artifacts:
+    paths:
+      - target/packages/*
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
 publish_crates:
  stage: build_packages
  needs:
-   - package_amd64_deb
+   - package_linux
  tags:
    - build-amd64-deb
  script:
@@ -215,7 +192,7 @@ publish_crates:
 publish_python:
  stage: build_packages
  needs:
-   - package_amd64_deb
+   - package_linux
  tags:
    - build-amd64-deb
  script:
@@ -227,8 +204,6 @@ publish_python:
 
 build_repositories:
   stage: distribute
-  #needs:
-  #  - publish_python
   tags:
     - build-orchestration
   variables:
@@ -239,26 +214,27 @@ build_repositories:
     - cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container
     - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~
     - bash scripts/cicd/build-orchestration/distribute-stable-packages.sh
+  dependencies:
+    - dryrun_package_linux
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
 delete_build_machines:
-  stage: distribute
-  needs:
-    - build_repositories
+  stage: cleanup
   tags:
     - build-orchestration
+  dependencies:
+    - create_build_machine
   script:
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-deb
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete arm64-deb
-    - bash scripts/cicd/build-orchestration/build-machine-ctrl.sh delete amd64-rpm
+    - uv --directory scripts/cicd-python sync
+    - uv --directory scripts/cicd-python run veilid_release_utils.py --delete-build-machine
   rules:
     - if: '$CI_COMMIT_TAG =~ /v\d.+/'
 
 # Dryrun release -- triggered by changes in .gitlab-ci.yml, CICD scripts, or Earthfile
 
-dryrun_create_build_machines:
-  stage: build_packages
+dryrun_create_build_machine:
+  stage: prepare
   tags:
     - build-orchestration
   script:
@@ -273,8 +249,6 @@ dryrun_create_build_machines:
 
 dryrun_package_linux:
   stage: build_packages
-  needs:
-    - dryrun_create_build_machines
   tags:
     - build-amd64-deb
   script:
@@ -289,7 +263,7 @@ dryrun_package_linux:
 dryrun_publish_crates:
   stage: build_packages
   needs:
-    - dryrun_create_build_machines
+    - dryrun_package_linux
   tags:
     - build-amd64-deb
   script:
@@ -301,7 +275,7 @@ dryrun_publish_crates:
 dryrun_publish_python:
   stage: build_packages
   needs:
-    - dryrun_create_build_machines
+    - dryrun_package_linux
   tags:
     - build-amd64-deb
   script:
@@ -321,7 +295,8 @@ dryrun_build_repositories:
     - cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container
     - cp scripts/cicd/build-orchestration/generate-stable-release.sh ~
     - bash scripts/cicd/build-orchestration/distribute-stable-packages.sh
-  dependencies: [dryrun_package_linux]
+  dependencies:
+    - dryrun_package_linux
   rules:
     - if: $CI_COMMIT_MESSAGE =~ /\[ci dryrun]/
 
@@ -341,7 +316,7 @@ dryrun_delete_build_machines:
   tags:
     - build-orchestration
   dependencies:
-    - dryrun_create_build_machines
+    - dryrun_create_build_machine
   script:
     - uv --directory scripts/cicd-python sync
     - uv --directory scripts/cicd-python run veilid_release_utils.py --delete-build-machine
@@ -352,7 +327,7 @@ dryrun_delete_build_machines:
 # Nightly build pipeline
 
 nightly_create_build_machines:
-  stage: build_packages
+  stage: prepare
   tags:
     - build-orchestration
   script:
@@ -367,8 +342,6 @@ nightly_create_build_machines:
 
 nightly_package_linux:
   stage: build_packages
-  needs:
-    - nightly_create_build_machines
   tags:
     - build-amd64-deb
   script:
@@ -380,32 +353,6 @@ nightly_package_linux:
   rules:
     - if: $IS_NIGHTLY == "true"
 
-# nightly_package_arm64_deb:
-#   stage: build_packages
-#   needs:
-#     - nightly_create_build_machines
-#   tags:
-#     - build-arm64-deb
-#   script:
-#     - earthly bootstrap
-#     - earthly +package-linux-arm64-deb --IS_NIGHTLY="$IS_NIGHTLY"
-#     - bash scripts/cicd/build-machine/scp-arm64-debs-to-orchestrator.sh
-#   rules:
-#     - if: $IS_NIGHTLY == "true"
-
-# nightly_package_amd64_rpm:
-#   stage: build_packages
-#   needs:
-#     - nightly_create_build_machines
-#   tags:
-#     - build-amd64-rpm
-#   script:
-#     - earthly bootstrap
-#     - earthly +package-linux-amd64-rpm --IS_NIGHTLY="$IS_NIGHTLY"
-#     - bash scripts/cicd/build-machine/scp-amd64-rpms-to-orchestrator.sh
-#   rules:
-#     - if: $IS_NIGHTLY == "true"
-
 nightly_build_repositories:
   stage: distribute
   tags:
@@ -418,7 +365,8 @@ nightly_build_repositories:
     - cp scripts/cicd/build-orchestration/rpm-repo-building/repobuild.sh ~/rpm-build-container
     - cp scripts/cicd/build-orchestration/generate-nightly-release.sh ~
     - bash scripts/cicd/build-orchestration/distribute-nightly-packages.sh
-  dependencies: [nightly_package_linux]
+  dependencies:
+    - nightly_package_linux
   rules:
     - if: $IS_NIGHTLY == "true"
 
@@ -441,7 +389,7 @@ stable_failed_pipeline_actions:
   tags:
     - build-orchestration
   dependencies:
-    - create_build_machines
+    - create_build_machine
   script:
     - uv --directory scripts/cicd-python sync
     - uv --directory scripts/cicd-python run veilid_release_utils.py --delete-build-machine

veilid-core/src/table_store/mod.rs

@@ -15,6 +15,7 @@ mod native;
 use native::*;
 
 use keyvaluedb::*;
+use weak_table::WeakValueHashMap;
 
 impl_veilid_log_facility!("tstore");
 
@@ -72,7 +73,7 @@ pub struct TableInfo {
 
 #[must_use]
 struct TableStoreInner {
-    opened: BTreeMap<String, Weak<TableDBUnlockedInner>>,
+    opened: WeakValueHashMap<String, Weak<TableDBUnlockedInner>>,
     encryption_key: Option<TypedSharedSecret>,
     all_table_names: HashMap<String, String>,
     all_tables_db: Option<Database>,
@@ -115,7 +116,7 @@ impl_veilid_component!(TableStore);
 impl TableStore {
     fn new_inner() -> TableStoreInner {
         TableStoreInner {
-            opened: BTreeMap::new(),
+            opened: WeakValueHashMap::new(),
             encryption_key: None,
             all_table_names: HashMap::new(),
             all_tables_db: None,
@@ -528,6 +529,7 @@ impl TableStore {
         self.flush().await;
 
         let mut inner = self.inner.lock();
+        inner.opened.shrink_to_fit();
         if !inner.opened.is_empty() {
             panic!(
                 "all open databases should have been closed: {:?}",
@@ -539,15 +541,6 @@ impl TableStore {
         inner.encryption_key = None;
     }
 
-    #[instrument(level = "trace", target = "tstore", skip_all)]
-    pub(crate) fn on_table_db_drop(&self, table: String) {
-        veilid_log!(self trace "dropping table db: {}", table);
-        let mut inner = self.inner.lock();
-        if inner.opened.remove(&table).is_none() {
-            unreachable!("should have removed an item");
-        }
-    }
-
     /// Get or create a TableDB database table. If the column count is greater than an
     /// existing TableDB's column count, the database will be upgraded to add the missing columns.
     #[instrument(level = "trace", target = "tstore", skip_all)]
@@ -566,25 +559,20 @@ impl TableStore {
 
         // See if this table is already opened, if so the column count must be the same
         {
-            let mut inner = self.inner.lock();
-            if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
-                match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone(), column_count) {
-                    Some(tdb) => {
-                        // Ensure column count isnt bigger
-                        let existing_col_count = tdb.get_column_count()?;
-                        if column_count > existing_col_count {
-                            return Err(VeilidAPIError::generic(format!(
-                                "database must be closed before increasing column count {} -> {}",
-                                existing_col_count, column_count,
-                            )));
-                        }
+            let inner = self.inner.lock();
+            if let Some(table_db_unlocked_inner) = inner.opened.get(&table_name) {
+                let tdb = TableDB::new_from_unlocked_inner(table_db_unlocked_inner, column_count);
 
-                        return Ok(tdb);
-                    }
-                    None => {
-                        inner.opened.remove(&table_name);
-                    }
-                };
+                // Ensure column count isnt bigger
+                let existing_col_count = tdb.get_column_count()?;
+                if column_count > existing_col_count {
+                    return Err(VeilidAPIError::generic(format!(
+                        "database must be closed before increasing column count {} -> {}",
+                        existing_col_count, column_count,
+                    )));
+                }
+
+                return Ok(tdb);
             }
         }
 
@@ -637,7 +625,7 @@ impl TableStore {
         // Keep track of opened DBs
         inner
             .opened
-            .insert(table_name.clone(), table_db.weak_unlocked_inner());
+            .insert(table_name.clone(), table_db.unlocked_inner());
 
         Ok(table_db)
     }

veilid-core/src/table_store/table_db.rs

@@ -38,13 +38,6 @@ impl fmt::Debug for TableDBUnlockedInner {
     }
 }
 
-impl Drop for TableDBUnlockedInner {
-    fn drop(&mut self) {
-        let table_store = self.registry.table_store();
-        table_store.on_table_db_drop(self.table.clone());
-    }
-}
-
 #[derive(Debug, Clone)]
 #[must_use]
 pub struct TableDB {
@@ -87,26 +80,24 @@ impl TableDB {
         }
     }
 
-    pub(super) fn try_new_from_weak_inner(
-        weak_inner: Weak<TableDBUnlockedInner>,
+    pub(super) fn new_from_unlocked_inner(
+        unlocked_inner: Arc<TableDBUnlockedInner>,
         opened_column_count: u32,
-    ) -> Option<Self> {
-        weak_inner.upgrade().map(|table_db_unlocked_inner| {
-            let db = &table_db_unlocked_inner.database;
-            let total_columns = db.num_columns().unwrap();
-            Self {
-                opened_column_count: if opened_column_count == 0 {
-                    total_columns
-                } else {
-                    opened_column_count
-                },
-                unlocked_inner: table_db_unlocked_inner,
-            }
-        })
+    ) -> Self {
+        let db = &unlocked_inner.database;
+        let total_columns = db.num_columns().unwrap();
+        Self {
+            opened_column_count: if opened_column_count == 0 {
+                total_columns
+            } else {
+                opened_column_count
+            },
+            unlocked_inner,
+        }
     }
 
-    pub(super) fn weak_unlocked_inner(&self) -> Weak<TableDBUnlockedInner> {
-        Arc::downgrade(&self.unlocked_inner)
+    pub(super) fn unlocked_inner(&self) -> Arc<TableDBUnlockedInner> {
+        self.unlocked_inner.clone()
     }
 
     /// Get the internal name of the table