filter for lost, fixed loading vptree

server.js

@@ -10,18 +10,20 @@ const port = 2000;
 app.use(bodyParser.json());
 
 // Function to load JSON data
-function loadJSON(filePath, callback) {
+function loadJSON(filePath) {
+    return new Promise((resolve, reject) => {
         fs.readFile(filePath, 'utf8', (err, data) => {
             if (err) {
-            return callback(err, null);
+                return reject(err);
             }
             try {
                 const jsonData = JSON.parse(data);
-            callback(null, jsonData);
+                resolve(jsonData);
             } catch (parseError) {
-            callback(parseError, null);
+                reject(parseError);
             }
         });
+    });
 }
 
 function cosineDistanceMatching(vector1, vector2) {
@@ -30,112 +32,105 @@ function cosineDistanceMatching(vector1, vector2) {
     return Math.sqrt(distance);
 }
 
-let tree, vectors, jsonEmbeddings
+let tree, vectors, jsonEmbeddings, filteredEmbeddings;
-
-buildVPTree()
-loadVPTree()
-
-// Load tree endpoint
-app.post('/search/', (req, res) => {
-    const { vector } = req.body;
-
-    console.log(vector)
-
-    if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) {
-        return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' });
-    }
-
-    let idx = queryVPTree(vector)
-
-    res.send({folder: jsonEmbeddings[idx].folder, image: jsonEmbeddings[idx].image, video: jsonEmbeddings[idx].video, frame: jsonEmbeddings[idx].frame});
-});
-
-function queryVPTree(value){
-    let nearest = tree.search(value)
-    let index = nearest[0].i
-    return index
-}
 
-function buildVPTree(){
+async function buildVPTree() {
     // Load JSON data from embeddings.json
     const jsonFilePath = path.join(__dirname, 'embeddings.json');
-    loadJSON(jsonFilePath, (err, jsonData) => {
+    try {
-            if (err) {
+        jsonEmbeddings = await loadJSON(jsonFilePath);
-                console.error('Error loading JSON:', err);
-                return;
-            }
-        // Extract vectors from the JSON data
-        jsonEmbeddings = jsonData
-        vectors = jsonEmbeddings.map(item => item.vector);
-        tree = vptree.build(vectors, cosineDistanceMatching)
-        const treeString = tree.stringify();
-        const fileName = "tree"
-        const filePath = path.join(__dirname, `${fileName}.txt`);
-
-        fs.writeFile(filePath, treeString, 'utf8', (err) => {
-            if (err) {
-                console.log("Tree did not save to file.")
-                return
-            }
-            
-            console.log("Tree saved successfully.")
-        });
-    });
-    
-}
 
-function saveVPTree(fileName){
+        // Filter embeddings
+        filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1);
+        vectors = filteredEmbeddings.map(item => item.vector);
 
+        tree = vptree.build(vectors, cosineDistanceMatching);
         const treeString = tree.stringify();
-
+        const fileName = "sv-tree";
         const filePath = path.join(__dirname, `${fileName}.txt`);
 
-    fs.writeFile(filePath, treeString, 'utf8', (err) => {
+        await fs.promises.writeFile(filePath, treeString, 'utf8');
-        if (err) {
+        console.log("Tree saved successfully.");
-            console.log("Tree did not save to file.")
+    } catch (err) {
-            return
+        console.error('Error building VP-tree:', err);
     }
-        
-        console.log("Tree saved successfully.")
-    });
 }
 
-function loadVPTree(){
+async function loadVPTree() {
-    loadTreeFromDisk("tree", (err, treeData) => {
+    try {
-        if (err) {
+        const treeData = await loadTreeFromDisk("sv-tree");
-            console.error('Failed to load tree from disk:', err);
-            return;
-        }
-
         // Rebuild the VP-tree using the saved structure
+        const jsonFilePath = path.join(__dirname, 'embeddings.json');
         try {
+            jsonEmbeddings = await loadJSON(jsonFilePath);
+            filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1); // Filter again after loading
+            vectors = filteredEmbeddings.map(item => item.vector);
+
+            console.log(vectors.length);
             tree = vptree.load(vectors, cosineDistanceMatching, treeData);
             console.log('Tree loaded successfully.');
-
         } catch (loadError) {
             console.error('Error loading the VP-tree:', loadError);
         }
-    });
+    } catch (err) {
+        console.error('Failed to load tree from disk:', err);
+    }
 }
 
-function loadTreeFromDisk(fileName, callback){
+function loadTreeFromDisk(fileName) {
     const filePath = path.join(__dirname, `${fileName}.txt`);
-
+    return new Promise((resolve, reject) => {
         fs.readFile(filePath, 'utf8', (err, data) => {
             if (err) {
                 console.error('Error reading the file:', err);
-            return callback(err, null);
+                return reject(err);
             }
 
             // Parse the data as a JavaScript object
             try {
                 const treeData = eval(`(${data})`); // Using eval to convert to object
-            callback(null, treeData);
+                resolve(treeData);
             } catch (parseError) {
                 console.error('Error parsing the tree data:', parseError);
-            callback(parseError, null);
+                reject(parseError);
+            }
+        });
+    });
+}
+
+async function initialize() {
+    await buildVPTree();
+    await loadVPTree();
+}
+
+// Initialize the VP-tree building and loading process
+initialize();
+
+app.post('/search/', (req, res) => {
+    const { vector } = req.body;
+
+    if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) {
+        return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' });
     }
+
+    let idx = queryVPTree(vector);
+    if (idx !== undefined && idx < filteredEmbeddings.length) {
+        res.send({ 
+            folder: filteredEmbeddings[idx].folder, 
+            image: filteredEmbeddings[idx].image, 
+            video: filteredEmbeddings[idx].video, 
+            frame: filteredEmbeddings[idx].frame, 
+            lost: filteredEmbeddings[idx].lost 
         });
+    } else {
+        res.status(404).send({ success: false, message: 'No matching entry found.' });
+    }
+});
+
+function queryVPTree(value) {
+    let nearest = tree.search(value);
+    let index = nearest[0].i;
+    return index;
 }
 
 app.listen(port, () => {

vp-tree-new.txt

@@ -1 +0,0 @@
-{i:4,m:7,M:9,μ:9,L:{i:6,m:7,M:8,μ:8,L:{i:8},R:{i:9,m:7,M:7,μ:7,R:{i:5}}},R:{i:2,m:8,M:8,μ:8,L:{i:0,m:7,M:7,μ:7,R:{i:3}},R:{i:7,m:7,M:7,μ:7,R:{i:1}}}}

vp-tree.txt

@@ -1 +0,0 @@
-{i:6,m:6,M:9,μ:8,L:{i:3,m:4,M:9,μ:7,L:{i:8},R:{i:4,m:9,M:9,μ:9,R:{i:0}}},R:{i:7,m:7,M:8,μ:8,L:{i:9,m:6,M:6,μ:6,R:{i:1}},R:{i:2,m:8,M:8,μ:8,R:{i:5}}}}