cailean
/
vp-tree-node
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

filter for lost, fixed loading vptree

master
cailean 3 weeks ago
parent
4124f0352d
commit
7b5a6edc22
6 changed files with 48124 additions and 1891 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 49826
      embeddings.json
  2. 185
      server.js
  3. 1
      sv-tree.txt
  4. 1
      tree.txt
  5. 1
      vp-tree-new.txt
  6. 1
      vp-tree.txt

49826
embeddings.json
View File

File diff suppressed because it is too large

185
server.js
View File

@ -10,17 +10,19 @@ const port = 2000;
app.use(bodyParser.json());
// Function to load JSON data
function loadJSON(filePath, callback) {
fs.readFile(filePath, 'utf8', (err, data) => {
if (err) {
return callback(err, null);
}
try {
const jsonData = JSON.parse(data);
callback(null, jsonData);
} catch (parseError) {
callback(parseError, null);
}
function loadJSON(filePath) {
return new Promise((resolve, reject) => {
fs.readFile(filePath, 'utf8', (err, data) => {
if (err) {
return reject(err);
}
try {
const jsonData = JSON.parse(data);
resolve(jsonData);
} catch (parseError) {
reject(parseError);
}
});
});
}
@ -30,114 +32,107 @@ function cosineDistanceMatching(vector1, vector2) {
return Math.sqrt(distance);
}
let tree, vectors, jsonEmbeddings
let tree, vectors, jsonEmbeddings, filteredEmbeddings;
buildVPTree()
loadVPTree()
async function buildVPTree() {
// Load JSON data from embeddings.json
const jsonFilePath = path.join(__dirname, 'embeddings.json');
try {
jsonEmbeddings = await loadJSON(jsonFilePath);
// Load tree endpoint
app.post('/search/', (req, res) => {
const { vector } = req.body;
// Filter embeddings
filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1);
vectors = filteredEmbeddings.map(item => item.vector);
console.log(vector)
tree = vptree.build(vectors, cosineDistanceMatching);
const treeString = tree.stringify();
const fileName = "sv-tree";
const filePath = path.join(__dirname, `${fileName}.txt`);
if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) {
return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' });
await fs.promises.writeFile(filePath, treeString, 'utf8');
console.log("Tree saved successfully.");
} catch (err) {
console.error('Error building VP-tree:', err);
}
}
let idx = queryVPTree(vector)
res.send({folder: jsonEmbeddings[idx].folder, image: jsonEmbeddings[idx].image, video: jsonEmbeddings[idx].video, frame: jsonEmbeddings[idx].frame});
});
async function loadVPTree() {
try {
const treeData = await loadTreeFromDisk("sv-tree");
// Rebuild the VP-tree using the saved structure
const jsonFilePath = path.join(__dirname, 'embeddings.json');
try {
jsonEmbeddings = await loadJSON(jsonFilePath);
filteredEmbeddings = jsonEmbeddings.filter(item => item.lost !== 1); // Filter again after loading
vectors = filteredEmbeddings.map(item => item.vector);
function queryVPTree(value){
let nearest = tree.search(value)
let index = nearest[0].i
return index
console.log(vectors.length);
tree = vptree.load(vectors, cosineDistanceMatching, treeData);
console.log('Tree loaded successfully.');
} catch (loadError) {
console.error('Error loading the VP-tree:', loadError);
}
} catch (err) {
console.error('Failed to load tree from disk:', err);
}
}
function buildVPTree(){
// Load JSON data from embeddings.json
const jsonFilePath = path.join(__dirname, 'embeddings.json');
loadJSON(jsonFilePath, (err, jsonData) => {
function loadTreeFromDisk(fileName) {
const filePath = path.join(__dirname, `${fileName}.txt`);
return new Promise((resolve, reject) => {
fs.readFile(filePath, 'utf8', (err, data) => {
if (err) {
console.error('Error loading JSON:', err);
return;
console.error('Error reading the file:', err);
return reject(err);
}
// Extract vectors from the JSON data
jsonEmbeddings = jsonData
vectors = jsonEmbeddings.map(item => item.vector);
tree = vptree.build(vectors, cosineDistanceMatching)
const treeString = tree.stringify();
const fileName = "tree"
const filePath = path.join(__dirname, `${fileName}.txt`);
fs.writeFile(filePath, treeString, 'utf8', (err) => {
if (err) {
console.log("Tree did not save to file.")
return
// Parse the data as a JavaScript object
try {
const treeData = eval(`(${data})`); // Using eval to convert to object
resolve(treeData);
} catch (parseError) {
console.error('Error parsing the tree data:', parseError);
reject(parseError);
}
console.log("Tree saved successfully.")
});
});
}
function saveVPTree(fileName){
const treeString = tree.stringify();
const filePath = path.join(__dirname, `${fileName}.txt`);
fs.writeFile(filePath, treeString, 'utf8', (err) => {
if (err) {
console.log("Tree did not save to file.")
return
}
console.log("Tree saved successfully.")
});
async function initialize() {
await buildVPTree();
await loadVPTree();
}
function loadVPTree(){
loadTreeFromDisk("tree", (err, treeData) => {
if (err) {
console.error('Failed to load tree from disk:', err);
return;
}
// Rebuild the VP-tree using the saved structure
try {
tree = vptree.load(vectors, cosineDistanceMatching, treeData);
console.log('Tree loaded successfully.');
// Initialize the VP-tree building and loading process
initialize();
} catch (loadError) {
console.error('Error loading the VP-tree:', loadError);
}
});
}
app.post('/search/', (req, res) => {
const { vector } = req.body;
function loadTreeFromDisk(fileName, callback){
const filePath = path.join(__dirname, `${fileName}.txt`);
if (!Array.isArray(vector) || vector.length !== 7 || !vector.every(num => typeof num === 'number')) {
return res.status(400).send({ success: false, message: 'Invalid vector provided. It must be an array of 7 floating-point numbers.' });
}
fs.readFile(filePath, 'utf8', (err, data) => {
if (err) {
console.error('Error reading the file:', err);
return callback(err, null);
}
let idx = queryVPTree(vector);
if (idx !== undefined && idx < filteredEmbeddings.length) {
res.send({
folder: filteredEmbeddings[idx].folder,
image: filteredEmbeddings[idx].image,
video: filteredEmbeddings[idx].video,
frame: filteredEmbeddings[idx].frame,
lost: filteredEmbeddings[idx].lost
});
} else {
res.status(404).send({ success: false, message: 'No matching entry found.' });
}
});
// Parse the data as a JavaScript object
try {
const treeData = eval(`(${data})`); // Using eval to convert to object
callback(null, treeData);
} catch (parseError) {
console.error('Error parsing the tree data:', parseError);
callback(parseError, null);
}
});
function queryVPTree(value) {
let nearest = tree.search(value);
let index = nearest[0].i;
return index;
}
app.listen(port, () => {
console.log(`Server listening at http://localhost:${port}`);
console.log(`Server listening at http://localhost:${port}`);
});

1
sv-tree.txt
View File

File diff suppressed because one or more lines are too long

1
tree.txt
View File

File diff suppressed because one or more lines are too long

1
vp-tree-new.txt
View File

@ -1 +0,0 @@
{i:4,m:7,M:9,μ:9,L:{i:6,m:7,M:8,μ:8,L:{i:8},R:{i:9,m:7,M:7,μ:7,R:{i:5}}},R:{i:2,m:8,M:8,μ:8,L:{i:0,m:7,M:7,μ:7,R:{i:3}},R:{i:7,m:7,M:7,μ:7,R:{i:1}}}}

1
vp-tree.txt
View File

@ -1 +0,0 @@
{i:6,m:6,M:9,μ:8,L:{i:3,m:4,M:9,μ:7,L:{i:8},R:{i:4,m:9,M:9,μ:9,R:{i:0}}},R:{i:7,m:7,M:8,μ:8,L:{i:9,m:6,M:6,μ:6,R:{i:1}},R:{i:2,m:8,M:8,μ:8,R:{i:5}}}}
Write Preview
Loading…
Cancel
Save