From 6e1cc2cbf0e458623f1c99462b40691b95409c7f Mon Sep 17 00:00:00 2001 From: anonpenguin Date: Thu, 10 Jul 2025 05:28:45 +0300 Subject: [PATCH] feat: Improve database query handling and slug generation in BlogAPIServer --- src/framework/models/BaseModel.ts | 17 +-- src/framework/query/QueryExecutor.ts | 113 ++++++++++++++++-- src/framework/services/OrbitDBService.ts | 5 - .../blog-scenario/docker/blog-api-server.ts | 40 +++++-- 4 files changed, 136 insertions(+), 39 deletions(-) diff --git a/src/framework/models/BaseModel.ts b/src/framework/models/BaseModel.ts index a7f2462..106075a 100644 --- a/src/framework/models/BaseModel.ts +++ b/src/framework/models/BaseModel.ts @@ -20,14 +20,11 @@ export abstract class BaseModel { static hooks: Map = new Map(); constructor(data: any = {}) { - console.log(`[DEBUG] Constructing ${this.constructor.name} with data:`, data); - // Generate ID first this.id = this.generateId(); // Apply field defaults first this.applyFieldDefaults(); - console.log(`[DEBUG] After applying defaults, instance properties:`, Object.getOwnPropertyNames(this)); // Then apply provided data, but only for properties that are explicitly provided if (data && typeof data === 'object') { @@ -44,12 +41,10 @@ export abstract class BaseModel { // For model fields, store in private field const privateKey = `_${key}`; (this as any)[privateKey] = data[key]; - console.log(`[DEBUG] Set private field ${privateKey} = ${data[key]}`); } else { // For non-field properties, set directly try { (this as any)[key] = data[key]; - console.log(`[DEBUG] Set property ${key} = ${data[key]}`); } catch (error) { console.error(`Error setting property ${key}:`, error); } @@ -65,7 +60,6 @@ export abstract class BaseModel { // Remove any instance properties that might shadow prototype getters this.cleanupShadowingProperties(); - console.log(`[DEBUG] After cleanup, instance properties:`, Object.getOwnPropertyNames(this)); } private cleanupShadowingProperties(): void { @@ -460,10 +454,6 @@ export abstract class BaseModel { const errors: string[] = []; const modelClass = this.constructor as typeof BaseModel; - console.log(`[DEBUG] Validating model ${modelClass.name}`); - console.log(`[DEBUG] Available fields:`, Array.from(modelClass.fields.keys())); - console.log(`[DEBUG] Instance properties:`, Object.getOwnPropertyNames(this)); - // Validate each field using getter values (more reliable) for (const [fieldName, fieldConfig] of modelClass.fields) { const privateKey = `_${fieldName}`; @@ -473,8 +463,6 @@ export abstract class BaseModel { // Use the property value (getter) if available, otherwise use private value const value = propertyValue !== undefined ? propertyValue : privateValue; - console.log(`[DEBUG] Field ${fieldName}: privateKey=${privateKey}, privateValue=${privateValue}, propertyValue=${propertyValue}, finalValue=${value}, config=`, fieldConfig); - const fieldErrors = await this.validateField(fieldName, value, fieldConfig); errors.push(...fieldErrors); } @@ -482,7 +470,6 @@ export abstract class BaseModel { const result = { valid: errors.length === 0, errors }; if (!result.valid) { - console.log(`[DEBUG] Validation failed:`, errors); throw new ValidationError(errors); } @@ -931,9 +918,7 @@ export abstract class BaseModel { } static query(this: typeof BaseModel & (new (data?: any) => T)): any { - // Import dynamically to avoid circular dependency - const QueryBuilderModule = require('../query/QueryBuilder'); - const QueryBuilder = QueryBuilderModule.QueryBuilder; + // Use the imported QueryBuilder directly return new QueryBuilder(this); } diff --git a/src/framework/query/QueryExecutor.ts b/src/framework/query/QueryExecutor.ts index 4a361cb..ad61cb4 100644 --- a/src/framework/query/QueryExecutor.ts +++ b/src/framework/query/QueryExecutor.ts @@ -113,8 +113,6 @@ export class QueryExecutor { private async executeUserSpecificQuery(userFilter: QueryCondition): Promise { const userIds = userFilter.operator === 'userIn' ? userFilter.value : [userFilter.value]; - console.log(`👤 Querying user databases for ${userIds.length} users`); - const results: T[] = []; // Query each user's database in parallel @@ -127,7 +125,7 @@ export class QueryExecutor { return await this.queryDatabase(userDB, this.model.storeType); } catch (error) { - console.warn(`Failed to query user ${userId} database:`, error); + // Silently handle user database query failures return []; } }); @@ -143,8 +141,6 @@ export class QueryExecutor { } private async executeGlobalIndexQuery(): Promise { - console.log(`📇 Querying global index for ${this.model.name}`); - // Query global index for user-scoped models const globalIndexName = `${this.model.modelName}GlobalIndex`; const indexShards = this.framework.shardManager.getAllShards(globalIndexName); @@ -175,10 +171,45 @@ export class QueryExecutor { // It's expensive but ensures completeness console.warn(`⚠️ Executing expensive all-users query for ${this.model.name}`); - // This would require getting all user IDs from the directory - // For now, return empty array and log warning - console.warn('All-users query not implemented - please ensure global indexes are set up'); - return []; + try { + // Get all entity IDs from the directory shards + const entityIds = await this.getAllEntityIdsFromDirectory(); + + if (entityIds.length === 0) { + console.warn('No entities found in directory shards'); + return []; + } + + const results: T[] = []; + + // Query each entity's database in parallel (in batches to avoid overwhelming the system) + const batchSize = 10; + for (let i = 0; i < entityIds.length; i += batchSize) { + const batch = entityIds.slice(i, i + batchSize); + const batchPromises = batch.map(async (entityId: string) => { + try { + const entityDB = await this.framework.databaseManager.getUserDatabase( + entityId, + this.model.modelName, + ); + return await this.queryDatabase(entityDB, this.model.storeType); + } catch (error) { + // Silently handle entity database query failures + return []; + } + }); + + const batchResults = await Promise.all(batchPromises); + for (const entityResult of batchResults) { + results.push(...entityResult); + } + } + + return this.postProcessResults(results); + } catch (error) { + console.error('Error executing all-entities query:', error); + return []; + } } private async executeGlobalQuery(): Promise { @@ -192,8 +223,6 @@ export class QueryExecutor { } private async executeShardedQuery(): Promise { - console.log(`🔀 Executing sharded query for ${this.model.name}`); - const conditions = this.query.getConditions(); const shardingConfig = this.model.sharding!; @@ -616,6 +645,68 @@ export class QueryExecutor { }; } + private async getAllEntityIdsFromDirectory(): Promise { + const maxRetries = 3; + const baseDelay = 100; // ms + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const directoryShards = await this.framework.databaseManager.getGlobalDirectoryShards(); + const entityIds: string[] = []; + + // Query all directory shards in parallel + const shardPromises = directoryShards.map(async (shard: any, index: number) => { + try { + // For keyvalue stores, we need to get the keys (entity IDs), not values + const shardData = shard.all(); + const keys = Object.keys(shardData); + return keys; + } catch (error) { + console.warn(`Failed to read directory shard ${index}:`, error); + return []; + } + }); + + const shardResults = await Promise.all(shardPromises); + + // Flatten all entity IDs from all shards + for (const shardEntityIds of shardResults) { + entityIds.push(...shardEntityIds); + } + + // If we found entities, return them + if (entityIds.length > 0) { + console.log(`📂 Found ${entityIds.length} entities in directory shards`); + return entityIds; + } + + // If this is our last attempt, return empty array + if (attempt === maxRetries) { + console.warn('📂 No entities found in directory shards after all attempts'); + return []; + } + + // Wait before retry with exponential backoff + const delay = baseDelay * Math.pow(2, attempt); + console.log(`📂 No entities found, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries + 1})`); + await new Promise(resolve => setTimeout(resolve, delay)); + + } catch (error) { + console.error(`Error getting entity IDs from directory (attempt ${attempt + 1}):`, error); + + if (attempt === maxRetries) { + return []; + } + + // Wait before retry + const delay = baseDelay * Math.pow(2, attempt); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + return []; + } + private getFrameworkInstance(): any { const framework = (globalThis as any).__debrosFramework; if (!framework) { diff --git a/src/framework/services/OrbitDBService.ts b/src/framework/services/OrbitDBService.ts index 5562275..893fda4 100644 --- a/src/framework/services/OrbitDBService.ts +++ b/src/framework/services/OrbitDBService.ts @@ -36,11 +36,6 @@ export class FrameworkOrbitDBService { } async openDatabase(name: string, type: StoreType): Promise { - console.log('FrameworkOrbitDBService.openDatabase called with:', { name, type }); - console.log('this.orbitDBService:', this.orbitDBService); - console.log('typeof this.orbitDBService.openDB:', typeof this.orbitDBService.openDB); - console.log('this.orbitDBService methods:', Object.getOwnPropertyNames(Object.getPrototypeOf(this.orbitDBService))); - if (typeof this.orbitDBService.openDB !== 'function') { throw new Error(`openDB is not a function. Service type: ${typeof this.orbitDBService}, methods: ${Object.getOwnPropertyNames(Object.getPrototypeOf(this.orbitDBService))}`); } diff --git a/tests/real-integration/blog-scenario/docker/blog-api-server.ts b/tests/real-integration/blog-scenario/docker/blog-api-server.ts index 0e30a7a..bc97eee 100644 --- a/tests/real-integration/blog-scenario/docker/blog-api-server.ts +++ b/tests/real-integration/blog-scenario/docker/blog-api-server.ts @@ -300,6 +300,18 @@ class BlogAPIServer { this.app.post('/api/posts', async (req, res, next) => { try { const sanitizedData = BlogValidation.sanitizePostInput(req.body); + + // Generate slug if not provided + if (!sanitizedData.slug && sanitizedData.title) { + sanitizedData.slug = sanitizedData.title + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/--+/g, '-') + .replace(/^-|-$/g, ''); + console.log(`[${this.nodeId}] Generated slug: ${sanitizedData.slug}`); + } + BlogValidation.validatePost(sanitizedData); const post = await Post.create(sanitizedData); @@ -380,7 +392,9 @@ class BlogAPIServer { // Update post this.app.put('/api/posts/:id', async (req, res, next) => { try { - const post = await Post.findById(req.params.id); + const post = await Post.query() + .where('id', req.params.id) + .first(); if (!post) { return res.status(404).json({ error: 'Post not found', @@ -404,7 +418,9 @@ class BlogAPIServer { // Publish post this.app.post('/api/posts/:id/publish', async (req, res, next) => { try { - const post = await Post.findById(req.params.id); + const post = await Post.query() + .where('id', req.params.id) + .first(); if (!post) { return res.status(404).json({ error: 'Post not found', @@ -423,7 +439,9 @@ class BlogAPIServer { // Unpublish post this.app.post('/api/posts/:id/unpublish', async (req, res, next) => { try { - const post = await Post.findById(req.params.id); + const post = await Post.query() + .where('id', req.params.id) + .first(); if (!post) { return res.status(404).json({ error: 'Post not found', @@ -442,7 +460,9 @@ class BlogAPIServer { // Like post this.app.post('/api/posts/:id/like', async (req, res, next) => { try { - const post = await Post.findById(req.params.id); + const post = await Post.query() + .where('id', req.params.id) + .first(); if (!post) { return res.status(404).json({ error: 'Post not found', @@ -460,7 +480,9 @@ class BlogAPIServer { // View post (increment view count) this.app.post('/api/posts/:id/view', async (req, res, next) => { try { - const post = await Post.findById(req.params.id); + const post = await Post.query() + .where('id', req.params.id) + .first(); if (!post) { return res.status(404).json({ error: 'Post not found', @@ -516,7 +538,9 @@ class BlogAPIServer { // Approve comment this.app.post('/api/comments/:id/approve', async (req, res, next) => { try { - const comment = await Comment.findById(req.params.id); + const comment = await Comment.query() + .where('id', req.params.id) + .first(); if (!comment) { return res.status(404).json({ error: 'Comment not found', @@ -535,7 +559,9 @@ class BlogAPIServer { // Like comment this.app.post('/api/comments/:id/like', async (req, res, next) => { try { - const comment = await Comment.findById(req.params.id); + const comment = await Comment.query() + .where('id', req.params.id) + .first(); if (!comment) { return res.status(404).json({ error: 'Comment not found',