cailean
/
vp-tree-node
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

filter for lost, fixed loading vptree

master
cailean 3 weeks ago
parent
4124f0352d
commit
7b5a6edc22
6 changed files with 48124 additions and 1891 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 49826
      embeddings.json
  2. 185
      server.js
  3. 1
      sv-tree.txt
  4. 1
      tree.txt
  5. 1
      vp-tree-new.txt
  6. 1
      vp-tree.txt

49826
embeddings.json
View File

File diff suppressed because it is too large

185
server.js
View File

@ -10,17 +10,19 @@ const port = 2000;
app.use(bodyParser.json()); app.use(bodyParser.json());
// Function to load JSON data // Function to load JSON data
function loadJSON(filePath, callback) { function loadJSON(filePath) {
fs.readFile(filePath, 'utf8', (err, data) => { return new Promise((resolve, reject) => {
if (err) { fs.readFile(filePath, 'utf8', (err, data) => {
return callback(err, null); if (err) {
} return reject(err);
try { }
const jsonData = JSON.parse(data); try {
callback(null, jsonData); const jsonData = JSON.parse(data);
} catch (parseError) { resolve(jsonData);
callback(parseError, null); } catch (parseError) {
} reject(parseError);
}
});
}); });
} }
@ -30,114 +32,107 @@ function cosineDistanceMatching(vector1, vector2) {
return Math.sqrt(distance); return Math.sqrt(distance);
} }
let tree, vectors, jsonEmbeddings let tree, vectors, jsonEmbeddings, filteredEmbeddings;
buildVPTree() async function buildVPTree() {
loadVPTree() // Load JSON data from embeddings.json
const jsonFilePath = path.join(__dirname, 'embeddings.json');
try {
jsonEmbeddings = await loadJSON(jsonFilePath);
// Load tree endpoint // Filter embeddings
app.post('/search/', (req, res) => { filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1);
const { vector } = req.body; vectors = filteredEmbeddings.map(item => item.vector);
console.log(vector) tree = vptree.build(vectors, cosineDistanceMatching);
const treeString = tree.stringify();
const fileName = "sv-tree";
const filePath = path.join(__dirname, `${fileName}.txt`);
if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) { await fs.promises.writeFile(filePath, treeString, 'utf8');
return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' }); console.log("Tree saved successfully.");
} catch (err) {
console.error('Error building VP-tree:', err);
} }
}
let idx = queryVPTree(vector) async function loadVPTree() {
try {
res.send({folder: jsonEmbeddings[idx].folder, image: jsonEmbeddings[idx].image, video: jsonEmbeddings[idx].video, frame: jsonEmbeddings[idx].frame}); const treeData = await loadTreeFromDisk("sv-tree");
}); // Rebuild the VP-tree using the saved structure
const jsonFilePath = path.join(__dirname, 'embeddings.json');
try {
jsonEmbeddings = await loadJSON(jsonFilePath);
filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1); // Filter again after loading
vectors = filteredEmbeddings.map(item => item.vector);
function queryVPTree(value){ console.log(vectors.length);
let nearest = tree.search(value) tree = vptree.load(vectors, cosineDistanceMatching, treeData);
let index = nearest[0].i console.log('Tree loaded successfully.');
return index } catch (loadError) {
console.error('Error loading the VP-tree:', loadError);
}
} catch (err) {
console.error('Failed to load tree from disk:', err);
}
} }
function buildVPTree(){ function loadTreeFromDisk(fileName) {
// Load JSON data from embeddings.json const filePath = path.join(__dirname, `${fileName}.txt`);
const jsonFilePath = path.join(__dirname, 'embeddings.json'); return new Promise((resolve, reject) => {
loadJSON(jsonFilePath, (err, jsonData) => { fs.readFile(filePath, 'utf8', (err, data) => {
if (err) { if (err) {
console.error('Error loading JSON:', err); console.error('Error reading the file:', err);
return; return reject(err);
} }
// Extract vectors from the JSON data
jsonEmbeddings = jsonData
vectors = jsonEmbeddings.map(item => item.vector);
tree = vptree.build(vectors, cosineDistanceMatching)
const treeString = tree.stringify();
const fileName = "tree"
const filePath = path.join(__dirname, `${fileName}.txt`);
fs.writeFile(filePath, treeString, 'utf8', (err) => { // Parse the data as a JavaScript object
if (err) { try {
console.log("Tree did not save to file.") const treeData = eval(`(${data})`); // Using eval to convert to object
return resolve(treeData);
} catch (parseError) {
console.error('Error parsing the tree data:', parseError);
reject(parseError);
} }
console.log("Tree saved successfully.")
}); });
}); });
} }
function saveVPTree(fileName){ async function initialize() {
await buildVPTree();
const treeString = tree.stringify(); await loadVPTree();
const filePath = path.join(__dirname, `${fileName}.txt`);
fs.writeFile(filePath, treeString, 'utf8', (err) => {
if (err) {
console.log("Tree did not save to file.")
return
}
console.log("Tree saved successfully.")
});
} }
function loadVPTree(){ // Initialize the VP-tree building and loading process
loadTreeFromDisk("tree", (err, treeData) => { initialize();
if (err) {
console.error('Failed to load tree from disk:', err);
return;
}
// Rebuild the VP-tree using the saved structure
try {
tree = vptree.load(vectors, cosineDistanceMatching, treeData);
console.log('Tree loaded successfully.');
} catch (loadError) { app.post('/search/', (req, res) => {
console.error('Error loading the VP-tree:', loadError); const { vector } = req.body;
}
});
}
function loadTreeFromDisk(fileName, callback){ if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) {
const filePath = path.join(__dirname, `${fileName}.txt`); return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' });
}
fs.readFile(filePath, 'utf8', (err, data) => { let idx = queryVPTree(vector);
if (err) { if (idx !== undefined && idx < filteredEmbeddings.length) {
console.error('Error reading the file:', err); res.send({
return callback(err, null); folder: filteredEmbeddings[idx].folder,
} image: filteredEmbeddings[idx].image,
video: filteredEmbeddings[idx].video,
frame: filteredEmbeddings[idx].frame,
lost: filteredEmbeddings[idx].lost
});
} else {
res.status(404).send({ success: false, message: 'No matching entry found.' });
}
});
// Parse the data as a JavaScript object function queryVPTree(value) {
try { let nearest = tree.search(value);
const treeData = eval(`(${data})`); // Using eval to convert to object let index = nearest[0].i;
callback(null, treeData); return index;
} catch (parseError) {
console.error('Error parsing the tree data:', parseError);
callback(parseError, null);
}
});
} }
app.listen(port, () => { app.listen(port, () => {
console.log(`Server listening at http://localhost:${port}`); console.log(`Server listening at http://localhost:${port}`);
}); });

1
sv-tree.txt
View File

File diff suppressed because one or more lines are too long

1
tree.txt
View File

File diff suppressed because one or more lines are too long

1
vp-tree-new.txt
View File

@ -1 +0,0 @@
{i:4,m:7,M:9,μ:9,L:{i:6,m:7,M:8,μ:8,L:{i:8},R:{i:9,m:7,M:7,μ:7,R:{i:5}}},R:{i:2,m:8,M:8,μ:8,L:{i:0,m:7,M:7,μ:7,R:{i:3}},R:{i:7,m:7,M:7,μ:7,R:{i:1}}}}

1
vp-tree.txt
View File

@ -1 +0,0 @@
{i:6,m:6,M:9,μ:8,L:{i:3,m:4,M:9,μ:7,L:{i:8},R:{i:4,m:9,M:9,μ:9,R:{i:0}}},R:{i:7,m:7,M:8,μ:8,L:{i:9,m:6,M:6,μ:6,R:{i:1}},R:{i:2,m:8,M:8,μ:8,R:{i:5}}}}
Write Preview
Loading…
Cancel
Save