diff --git a/.gitignore b/.gitignore index 2cb5698..0d56570 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ wheels/ # Virtual environments .venv/ - +venv/ .env .dockerignore @@ -22,3 +22,6 @@ src/models/ *.log.* *.rdb *.db + +codexify.sqlite-shm +codexify.sqlite-wal \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c8dd4b9..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -services: - web: - build: . - ports: - - "8089:8089" - command: uvicorn main:app --host 0.0.0.0 --port 8089 - environment: - - REDIS_URL=redis://redis:6379/0 - - REDIS_PASSWORD=your_password - depends_on: - - redis - - worker: - build: . - command: rq worker - environment: - - REDIS_URL=redis://redis:6379/0 - - REDIS_PASSWORD=your_password - depends_on: - - redis - - redis: - image: redis:latest - ports: - - "6379:6379" - networks: - - app-network - - api: - # ... your API service configuration ... - environment: - - REDIS_HOST=redis # When using Docker, the host should be the service name - depends_on: - - redis - networks: - - app-network - -networks: - app-network: - driver: bridge diff --git a/requirements.txt b/requirements.txt index bf7aa83..de4b895 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ uvicorn uvloop zstandard rq==1.10.1 +greenlet diff --git a/src/functions.py b/src/functions.py index 061ec7f..4efc44f 100644 --- a/src/functions.py +++ b/src/functions.py @@ -412,11 +412,16 @@ async def get_texts_for_model_and_embedding_pooling_method(llm_model_name: str, ) return texts_by_model_and_embedding_pooling_method -async def get_or_compute_embedding(request: EmbeddingRequest, req: Request = None, client_ip: str = None, document_file_hash: str = None, use_verbose: bool = True) -> dict: - request_time = datetime.utcnow() # Capture request time as datetime object - ip_address = ( - client_ip or (req.client.host if req else "localhost") - ) # If client_ip is provided, use it; otherwise, try to get from req; if not available, default to "localhost" +async def get_or_compute_embedding( + request: EmbeddingRequest, + req: Request = None, + client_ip: str = None, + document_file_hash: str = None, + use_verbose: bool = True, + db_writer = None +) -> dict: + request_time = datetime.utcnow() + ip_address = client_ip or (req.client.host if req else None) if use_verbose: logger.info(f"Received request for embedding for '{request.text}' using model '{request.llm_model_name}' and embedding pooling method '{request.embedding_pooling_method}' from IP address '{ip_address}'") text_embedding_instance = await get_embedding_from_db( @@ -469,7 +474,10 @@ async def get_or_compute_embedding(request: EmbeddingRequest, req: Request = Non if word_length_of_input_text > 0: if use_verbose: logger.info(f"Embedding calculated for '{request.text}' using model '{request.llm_model_name}' and embedding pooling method '{request.embedding_pooling_method}' in {total_time:,.2f} seconds, or an average of {total_time/word_length_of_input_text :.2f} seconds per word. Now saving to database...") - await db_writer.enqueue_write([embedding_instance]) # Enqueue the write operation using the db_writer instance directly + # Use the passed db_writer if available + if db_writer: + await db_writer.enqueue_write([embedding_instance]) + return {"text_embedding_dict": embedding_instance.as_dict()} async def calculate_sentence_embeddings_list(llama, texts: list, embedding_pooling_method: str) -> list: diff --git a/src/main.py b/src/main.py index eb57fbe..0e550e4 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,3 @@ - from db import AsyncSessionLocal, create_async_engine, create_tables from utils import build_faiss_indexes, configure_redis_optimally from models import DocumentEmbedding, Document, TextEmbedding, DocumentContentResponse, DocumentPydantic, SemanticDataTypeResponse, AllSemanticDataTypesResponse @@ -498,77 +497,164 @@ async def simple_semantic_search(request: SemanticSearchRequest, req: Request, t logger.error(traceback.format_exc()) # Print the traceback raise HTTPException(status_code=500, detail="Internal Server Error") -@app.post("/semantic-search/advanced") -async def advanced_semantic_search(request: AdvancedSemanticSearchRequest, req: Request, token: str = None) -> AdvancedSemanticSearchResponse: - - global faiss_indexes, associated_texts_by_model_and_pooling_method - request_time = datetime.utcnow() - request.query_text = prepare_string_for_embedding(request.query_text) - unique_id = f"advanced_semantic_search_{request.query_text}_{request.llm_model_name}_{request.embedding_pooling_method}_{request.similarity_filter_percentage}_{request.number_of_most_similar_strings_to_return}" - - - faiss_indexes, associated_texts_by_model_and_pooling_method = await build_faiss_indexes(force_rebuild=True) +@app.post("/semantic-search/advanced", response_model=Dict[str, Any]) +async def advanced_semantic_search( + request: AdvancedSemanticSearchRequest, + token: str = None +) -> Dict[str, Any]: + """ + Queue an advanced semantic search request for processing by a worker. + """ try: - faiss_index = faiss_indexes[(request.llm_model_name, request.embedding_pooling_method)] - except KeyError: - raise HTTPException(status_code=400, detail=f"No FAISS index found for model: {request.llm_model_name} and pooling method: {request.embedding_pooling_method}") - llm_model_name = request.llm_model_name - embedding_pooling_method = request.embedding_pooling_method - num_results_before_corpus_filter = request.number_of_most_similar_strings_to_return*25 - logger.info(f"Received request to find most similar strings for query text: `{request.query_text}` using model: {llm_model_name}") + # Create a unique job ID + timestamp = datetime.now().isoformat() + unique_id = hashlib.md5(f"{request.query_text}_{request.llm_model_name}_{timestamp}".encode()).hexdigest() + + logger.info(f"Processing advanced semantic search request for query: {request.query_text}") + + # Check existing jobs using the global redis_manager + try: + document_scans_queue = redis_manager.get_queue('document_scans') + existing_jobs = document_scans_queue.get_job_ids() + for job_id in existing_jobs: + try: + job = Job.fetch(job_id, connection=redis_manager.redis_sync) + if (job and job.args and + job.args[6] == request.query_text and # query_text is the 7th argument + job.args[1] == request.llm_model_name and + job.args[2] == request.embedding_pooling_method and + job.get_status() != 'failed'): + return { + "status": "already_queued", + "message": f"Search already in progress. Job ID: {job_id}", + "job_id": job_id + } + except Exception as fetch_err: + logger.warning(f"Error checking existing job {job_id}: {str(fetch_err)}") + continue + + except Exception as e: + logger.warning(f"Error checking existing jobs: {str(e)}") + + # Enqueue the new task using the global redis_manager + try: + job = document_scans_queue.enqueue( + 'worker.scan_document_task', + args=( + None, # document_hash is no longer needed + request.llm_model_name, + request.embedding_pooling_method, + request.corpus_identifier_string, + request.json_format, + request.send_back_json_or_zip_file, + request.query_text, + request.similarity_filter_percentage, + request.number_of_most_similar_strings_to_return, + request.result_sorting_metric + ), + job_id=unique_id, + result_ttl=86400, + failure_ttl=86400, + timeout='1h' + ) + + if not job: + raise Exception("Job creation failed") + + logger.info(f"Job enqueued successfully. Job ID: {unique_id}") + + # Verify the job was enqueued + verification_attempts = 3 + for attempt in range(verification_attempts): + try: + enqueued_job = Job.fetch(unique_id, connection=redis_manager.redis_sync) + if enqueued_job: + return { + "status": "queued", + "message": f"Search queued successfully. Job ID: {unique_id}", + "job_id": unique_id + } + except Exception as fetch_err: + if attempt == verification_attempts - 1: + raise + logger.warning(f"Verification attempt {attempt + 1} failed: {str(fetch_err)}") + await asyncio.sleep(0.5) # Wait briefly before retrying + + raise Exception("Job verification failed after multiple attempts") + + except Exception as e: + logger.error(f"Failed to enqueue job: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to queue semantic search: {str(e)}" + ) + + except Exception as e: + error_msg = f"Failed to process semantic search request: {str(e)}" + logger.error(error_msg) + logger.error(traceback.format_exc()) + raise HTTPException(status_code=500, detail=error_msg) + +@app.get("/semantic-search/status/{job_id}", response_model=Dict[str, Any]) +async def get_search_status(job_id: str) -> Dict[str, Any]: + """ + Check the status of a semantic search job + """ try: - logger.info(f"Computing embedding for input text: {request.query_text}") - embedding_request = EmbeddingRequest(text=request.query_text, llm_model_name=llm_model_name, embedding_pooling_method=embedding_pooling_method) - embedding_response = await get_or_compute_embedding(embedding_request, req) - embedding_json = embedding_response["text_embedding_dict"]["embedding_json"] - embedding_vector = json.loads(embedding_json) - input_embedding = np.array(embedding_vector).astype('float32').reshape(1, -1) - faiss.normalize_L2(input_embedding) - logger.info(f"Computed embedding for input text: {request.query_text}") - final_results = [] - faiss_index = faiss_indexes[(llm_model_name, embedding_pooling_method)] - if faiss_index is None: - raise HTTPException(status_code=400, detail=f"No FAISS index found for model: {llm_model_name} and pooling method: {embedding_pooling_method}") - num_results = max([1, int((1 - request.similarity_filter_percentage) * len(associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method]))]) - num_results_before_corpus_filter = min(num_results_before_corpus_filter, len(associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method])) - similarities, indices = faiss_index.search(input_embedding, num_results_before_corpus_filter) - filtered_indices = indices[0] - filtered_similarities = similarities[0] - similarity_results = [] - associated_texts = associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method] - list_of_corpus_identifier_strings = await get_list_of_corpus_identifiers_from_list_of_embedding_texts(associated_texts, llm_model_name, embedding_pooling_method) - for idx, similarity in zip(filtered_indices, filtered_similarities): - if idx < len(associated_texts) and list_of_corpus_identifier_strings[idx] == request.corpus_identifier_string: - associated_text = associated_texts[idx] - similarity_results.append((similarity, associated_text)) - similarity_results = sorted(similarity_results, key=lambda x: x[0], reverse=True)[:num_results] - for _, associated_text in similarity_results: - embedding_request = EmbeddingRequest(text=associated_text, llm_model_name=llm_model_name, embedding_pooling_method=embedding_pooling_method) - embedding_response = await get_or_compute_embedding(request=embedding_request, req=req, use_verbose=False) - embedding_json = embedding_response["text_embedding_dict"]["embedding_json"] - embedding_vector = json.loads(embedding_json) - comparison__embedding = np.array(embedding_vector).astype('float32').reshape(1, -1) - params = { - "vector_1": input_embedding.tolist()[0], - "vector_2": comparison__embedding.tolist()[0], - "similarity_measure": "all" - } - similarity_stats_str = fvs.py_compute_vector_similarity_stats(json.dumps(params)) - similarity_stats_json = json.loads(similarity_stats_str) - final_results.append({ - "search_result_text": associated_text, - "similarity_to_query_text": similarity_stats_json + job = Job.fetch(job_id, connection=redis_manager.redis_sync) + + status_mapping = { + 'queued': {'status': 'pending', 'message': 'Search queued'}, + 'started': {'status': 'pending', 'message': 'Search in progress'}, + 'finished': {'status': 'completed', 'result': job.result}, + 'failed': {'status': 'failed', 'error': str(job.exc_info)}, + 'stopped': {'status': 'stopped', 'message': 'Search stopped'}, + 'deferred': {'status': 'pending', 'message': 'Search deferred'} + } + + job_status = job.get_status() + return status_mapping.get(job_status, { + 'status': 'unknown', + 'message': f'Unknown job status: {job_status}' + }) + + except Exception as e: + error_msg = f"Error fetching job status: {str(e)}" + logger.error(error_msg) + logger.error(traceback.format_exc()) + return { + "status": "error", + "message": error_msg + } + +@app.get("/semantic-search/jobs", response_model=Dict[str, Any]) +async def list_search_jobs() -> Dict[str, Any]: + """ + List all semantic search jobs and their statuses + """ + try: + jobs = [] + job_ids = Queue('document_scans', connection=redis_manager.redis_sync).get_job_ids() + + for job_id in job_ids: + job = Job.fetch(job_id, connection=redis_manager.redis_sync) + jobs.append({ + 'job_id': job_id, + 'status': job.get_status(), + 'created_at': job.created_at.isoformat() if job.created_at else None, + 'query_text': job.args[6] if job.args else None # query_text is the 7th argument }) - num_to_return = request.number_of_most_similar_strings_to_return if request.number_of_most_similar_strings_to_return is not None else len(final_results) - results = sorted(final_results, key=lambda x: x["similarity_to_query_text"][request.result_sorting_metric], reverse=True)[:num_to_return] - response_time = datetime.utcnow() - total_time = (response_time - request_time).total_seconds() - logger.info(f"Finished advanced search in {total_time} seconds. Found {len(results)} results.") - return {"query_text": request.query_text, "corpus_identifier_string": request.corpus_identifier_string, "embedding_pooling_method": request.embedding_pooling_method, "results": results} + + return { + "status": "success", + "jobs": jobs + } + except Exception as e: - logger.error(f"An error occurred while processing the request: {e}") - traceback.print_exc() - raise HTTPException(status_code=500, detail="Internal Server Error") + error_msg = f"Error listing jobs: {str(e)}" + logger.error(error_msg) + logger.error(traceback.format_exc()) + raise HTTPException(status_code=500, detail=error_msg) @app.post("/semantic-types/") async def create_new_semantic_data_type(request: SemanticDataTypeEmbeddingRequest, req: Request = None, token: str = None, client_ip: str = None, document_file_hash: str = None) -> SemanticDataTypeEmbeddingResponse: @@ -959,4 +1045,4 @@ async def delete_documents( if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8089) + uvicorn.run(app, host="0.0.0.0", port=8089) \ No newline at end of file diff --git a/src/model_urls.json b/src/model_urls.json index adb6824..de1ee98 100644 --- a/src/model_urls.json +++ b/src/model_urls.json @@ -1 +1 @@ -["https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q6_K.gguf", "https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf", "https://huggingface.co/vonjack/bge-m3-gguf/resolve/main/bge-m3-q8_0.gguf", "https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-gguf/resolve/main/llava-llama-3-8b-v1_1-int4.gguf"] \ No newline at end of file +["https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q6_K.gguf"] \ No newline at end of file diff --git a/src/models.py b/src/models.py index 3064cfc..ebe0c36 100644 --- a/src/models.py +++ b/src/models.py @@ -331,6 +331,9 @@ class AdvancedSemanticSearchRequest(BaseModel): similarity_filter_percentage: float = 0.01 number_of_most_similar_strings_to_return: int = 10 result_sorting_metric: str = "hoeffding_d" + json_format: str = "records" + send_back_json_or_zip_file: str = "json" + @field_validator('result_sorting_metric') def validate_similarity_measure(cls, value): valid_measures = ["spearman_rho", "kendall_tau", "approximate_distance_correlation", "jensen_shannon_dependency_measure", "hoeffding_d"] diff --git a/src/worker.log b/src/worker.log index 2cab3d2..fb4071d 100644 --- a/src/worker.log +++ b/src/worker.log @@ -6677,3 +6677,1477 @@ NameError: name 'urllib' is not defined. Did you forget to import 'urllib' 2024-10-27 13:34:46,478 - worker - INFO - Downloading model from https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-gguf/resolve/main/llava-llama-3-8b-v1_1-int4.gguf 2024-10-27 13:36:44,363 - rq.worker - INFO - model_downloads: Job OK (f5703c84282ef486a8e27f4c7d1dcefe) 2024-10-27 13:36:44,366 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 11:53:23,830 - main - INFO - Starting application initialization +2024-10-29 11:53:23,832 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 11:53:23,837 - db - INFO - Database initialization completed. +2024-10-29 11:53:23,849 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 29; Took 0.012242 seconds, for an average of 0.00042213793103448275 seconds per hash. +2024-10-29 11:53:23,849 - utils - INFO - Checking models directory... +2024-10-29 11:53:23,850 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 11:53:23,850 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 11:53:23,850 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 11:53:23,851 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 11:53:23,851 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 11:53:23,852 - utils - INFO - Model downloads completed. +2024-10-29 11:53:23,857 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 11:53:23,860 - main - INFO - Application initialization complete +2024-10-29 11:54:35,619 - __main__ - INFO - Initializing worker... +2024-10-29 11:54:35,620 - __main__ - INFO - Worker listening to queues: ['model_downloads', 'file_uploads', 'document_scans'] +2024-10-29 11:54:35,633 - __main__ - INFO - Worker started successfully with PID: 90125 +2024-10-29 11:54:37,311 - __mp_main__ - INFO - Connecting to Redis at localhost:6379 +2024-10-29 11:54:37,312 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 11:54:37,313 - __mp_main__ - ERROR - Redis connection error: the greenlet library is required to use this function. No module named 'greenlet' +2024-10-29 11:54:58,167 - __main__ - INFO - Initializing worker... +2024-10-29 11:54:58,167 - __main__ - INFO - Worker listening to queues: ['model_downloads', 'file_uploads', 'document_scans'] +2024-10-29 11:54:58,177 - __main__ - INFO - Worker started successfully with PID: 90242 +2024-10-29 11:54:59,429 - __mp_main__ - INFO - Connecting to Redis at localhost:6379 +2024-10-29 11:54:59,430 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 11:54:59,436 - db - INFO - Database initialization completed. +2024-10-29 11:54:59,450 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 29; Took 0.013596 seconds, for an average of 0.00046882758620689656 seconds per hash. +2024-10-29 11:54:59,451 - utils - INFO - Checking models directory... +2024-10-29 11:54:59,451 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 11:54:59,451 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 11:54:59,451 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 11:54:59,451 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 11:54:59,452 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 11:54:59,452 - utils - INFO - Model downloads completed. +2024-10-29 11:54:59,459 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 11:54:59,473 - rq.worker - INFO - Worker rq:worker:worker-90242: started, version 1.10.1 +2024-10-29 11:54:59,473 - rq.worker - INFO - Subscribing to channel rq:pubsub:worker-90242 +2024-10-29 11:54:59,474 - rq.worker - INFO - *** Listening on model_downloads, file_uploads, document_scans... +2024-10-29 11:54:59,475 - rq.scheduler - INFO - Trying to acquire locks for file_uploads, model_downloads, document_scans +2024-10-29 11:54:59,482 - rq.worker - INFO - Cleaning registries for queue: model_downloads +2024-10-29 11:54:59,484 - rq.worker - INFO - Cleaning registries for queue: file_uploads +2024-10-29 11:54:59,485 - rq.worker - INFO - Cleaning registries for queue: document_scans +2024-10-29 11:55:00,722 - rq.scheduler - INFO - Scheduler for document_scans,file_uploads,model_downloads started with PID 90244 +2024-10-29 11:56:10,892 - main - INFO - Processing model URL: https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 11:56:10,902 - main - INFO - Job enqueued successfully. Job ID: 7e202b4bbf88faab76aef5beafef9a01 +2024-10-29 11:56:10,905 - rq.worker - INFO - model_downloads: worker.download_model_task('https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/...) (7e202b4bbf88faab76aef5beafef9a01) +2024-10-29 11:56:10,932 - worker - INFO - Starting download task for model URL: https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 11:56:10,934 - utils - INFO - Model URL not found in database. Adding https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf now... +2024-10-29 11:56:10,935 - utils - INFO - Model URL added: https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 11:56:10,935 - worker - INFO - Downloading model from https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 11:58:29,522 - rq.worker - INFO - model_downloads: Job OK (7e202b4bbf88faab76aef5beafef9a01) +2024-10-29 11:58:29,523 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 11:59:12,963 - main - INFO - Received request to retrieve all stored documents +2024-10-29 11:59:12,963 - main - INFO - Retrieving all stored documents from the database +2024-10-29 11:59:13,006 - main - INFO - Retrieved 8 stored documents from the database +2024-10-29 11:59:23,091 - main - INFO - Processing document URL: string +2024-10-29 11:59:23,094 - main - INFO - Job enqueued successfully. Job ID: f7c4288550bd6ac722ce4e0d1730b9cd +2024-10-29 11:59:23,095 - rq.worker - INFO - file_uploads: worker.upload_file_task('/tmp/f7c4288550bd6ac722ce4e0d1730b9cd_note1.txt', 'string', 0, 'nomic-embed-text-v1.5.Q6_K', 'mean', '', 'records', 'zip', 'string') (f7c4288550bd6ac722ce4e0d1730b9cd) +2024-10-29 11:59:23,126 - worker - INFO - Starting file upload task for file: /tmp/f7c4288550bd6ac722ce4e0d1730b9cd_note1.txt +2024-10-29 11:59:23,129 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 11:59:23,139 - db - INFO - Database initialization completed. +2024-10-29 11:59:23,146 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 29; Took 0.006281 seconds, for an average of 0.00021658620689655173 seconds per hash. +2024-10-29 11:59:23,146 - utils - INFO - Checking models directory... +2024-10-29 11:59:23,147 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 11:59:23,150 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 11:59:23,151 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 11:59:23,152 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 11:59:23,153 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 11:59:23,154 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 11:59:23,154 - utils - INFO - Model downloads completed. +2024-10-29 11:59:23,195 - worker - INFO - Successfully saved document and embedding with hash: string +2024-10-29 11:59:23,197 - rq.worker - INFO - file_uploads: Job OK (f7c4288550bd6ac722ce4e0d1730b9cd) +2024-10-29 11:59:23,197 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:00:00,389 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:00:00,390 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:00:00,445 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:01:50,355 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:01:50,361 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 12:01:50,362 - main - INFO - Processed 1 semantic data types +2024-10-29 12:04:24,777 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:04:24,780 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:04:24,833 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:04:24,842 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 12:04:24,842 - main - INFO - Processed 1 semantic data types +2024-10-29 12:04:24,896 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:04:34,170 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:04:34,171 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:04:34,232 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:04:34,235 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:04:34,235 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:04:34,268 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:04:52,485 - main - INFO - Retrieving content for document with document_hash: string +2024-10-29 12:04:52,495 - main - INFO - Retrieved content for document with document_hash: string +2024-10-29 12:05:19,016 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:05:19,022 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 12:05:19,023 - main - INFO - Processed 1 semantic data types +2024-10-29 12:05:19,025 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:05:19,026 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 12:05:19,026 - main - INFO - Processed 1 semantic data types +2024-10-29 12:05:31,481 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'carmel friends...' +2024-10-29 12:05:31,482 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.5 seconds. +2024-10-29 12:05:31,482 - functions - INFO - That's an average of 482.87 ms per sentence and 2.071 sentences per second (and 0.0290 total characters per ms) using pooling method 'mean' +2024-10-29 12:05:31,503 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'rohit...' +2024-10-29 12:05:31,503 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.0 seconds. +2024-10-29 12:05:31,503 - functions - INFO - That's an average of 17.16 ms per sentence and 58.265 sentences per second (and 0.2913 total characters per ms) using pooling method 'mean' +2024-10-29 12:05:31,519 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'mustafa...' +2024-10-29 12:05:31,519 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.0 seconds. +2024-10-29 12:05:31,519 - functions - INFO - That's an average of 11.55 ms per sentence and 86.573 sentences per second (and 0.6060 total characters per ms) using pooling method 'mean' +2024-10-29 12:05:31,529 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'jerome...' +2024-10-29 12:05:31,530 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.0 seconds. +2024-10-29 12:05:31,530 - functions - INFO - That's an average of 7.96 ms per sentence and 125.644 sentences per second (and 0.7539 total characters per ms) using pooling method 'mean' +2024-10-29 12:05:35,593 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:05:35,595 - main - INFO - Retrieved 2 semantic data types from the database +2024-10-29 12:05:35,595 - main - INFO - Processed 2 semantic data types +2024-10-29 12:05:35,597 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:05:35,598 - main - INFO - Retrieved 2 semantic data types from the database +2024-10-29 12:05:35,598 - main - INFO - Processed 2 semantic data types +2024-10-29 12:05:49,483 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:05:49,484 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:05:49,520 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:05:49,522 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:05:49,522 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:05:49,550 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:06:01,232 - main - INFO - Processing document URL: None +2024-10-29 12:06:01,235 - main - INFO - Job enqueued successfully. Job ID: 0f967c7c0292734b320370d98c84b133 +2024-10-29 12:06:01,237 - rq.worker - INFO - file_uploads: worker.upload_file_task('/tmp/0f967c7c0292734b320370d98c84b133_note2.txt', None, None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergynotes', 'records', 'zip', None) (0f967c7c0292734b320370d98c84b133) +2024-10-29 12:06:01,250 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:06:01,251 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:06:01,277 - main - INFO - Retrieved 9 stored documents from the database +2024-10-29 12:06:01,290 - worker - INFO - Starting file upload task for file: /tmp/0f967c7c0292734b320370d98c84b133_note2.txt +2024-10-29 12:06:01,292 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:06:01,310 - db - INFO - Database initialization completed. +2024-10-29 12:06:01,318 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 33; Took 0.007 seconds, for an average of 0.00021212121212121213 seconds per hash. +2024-10-29 12:06:01,319 - utils - INFO - Checking models directory... +2024-10-29 12:06:01,319 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:06:01,320 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:06:01,321 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:06:01,321 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:06:01,321 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:06:01,322 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:06:01,322 - utils - INFO - Model downloads completed. +2024-10-29 12:06:01,342 - worker - INFO - Successfully saved document and embedding with hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:06:01,345 - rq.worker - INFO - file_uploads: Job OK (0f967c7c0292734b320370d98c84b133) +2024-10-29 12:06:01,345 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:06:06,200 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:06:06,200 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:06:06,239 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:06:06,241 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:06:06,241 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:06:06,268 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:06:17,712 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:06:17,712 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:06:17,737 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:06:17,740 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:06:17,740 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:06:17,768 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:07:33,583 - main - INFO - Retrieving content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:07:33,594 - main - INFO - Retrieved content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:15:52,367 - main - INFO - Application shutdown initiated +2024-10-29 12:15:55,823 - main - INFO - Starting application initialization +2024-10-29 12:15:55,824 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:15:55,829 - db - INFO - Database initialization completed. +2024-10-29 12:15:55,840 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.0108 seconds, for an average of 0.00031764705882352944 seconds per hash. +2024-10-29 12:15:55,840 - utils - INFO - Checking models directory... +2024-10-29 12:15:55,840 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:15:55,841 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:15:55,841 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:15:55,841 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:15:55,841 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:15:55,841 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:15:55,841 - utils - INFO - Model downloads completed. +2024-10-29 12:15:55,847 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:15:55,849 - main - INFO - Application initialization complete +2024-10-29 12:15:57,975 - main - INFO - Application shutdown initiated +2024-10-29 12:16:00,338 - main - INFO - Starting application initialization +2024-10-29 12:16:00,341 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:16:00,346 - db - INFO - Database initialization completed. +2024-10-29 12:16:00,363 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.016393 seconds, for an average of 0.0004821470588235294 seconds per hash. +2024-10-29 12:16:00,363 - utils - INFO - Checking models directory... +2024-10-29 12:16:00,363 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:16:00,363 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:16:00,363 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:16:00,364 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:16:00,364 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:16:00,364 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:16:00,364 - utils - INFO - Model downloads completed. +2024-10-29 12:16:00,369 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:16:00,370 - main - INFO - Application initialization complete +2024-10-29 12:16:03,507 - main - INFO - Application shutdown initiated +2024-10-29 12:16:05,759 - main - INFO - Starting application initialization +2024-10-29 12:16:05,760 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:16:05,764 - db - INFO - Database initialization completed. +2024-10-29 12:16:05,774 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.010509 seconds, for an average of 0.00030908823529411764 seconds per hash. +2024-10-29 12:16:05,774 - utils - INFO - Checking models directory... +2024-10-29 12:16:05,774 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:16:05,775 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:16:05,775 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:16:05,775 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:16:05,775 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:16:05,775 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:16:05,775 - utils - INFO - Model downloads completed. +2024-10-29 12:16:05,780 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:16:05,781 - main - INFO - Application initialization complete +2024-10-29 12:19:31,495 - rq.worker - INFO - Cleaning registries for queue: model_downloads +2024-10-29 12:19:31,500 - rq.worker - INFO - Cleaning registries for queue: file_uploads +2024-10-29 12:19:31,501 - rq.worker - INFO - Cleaning registries for queue: document_scans +2024-10-29 12:20:33,257 - main - INFO - Application shutdown initiated +2024-10-29 12:24:45,735 - main - INFO - Starting application initialization +2024-10-29 12:24:45,736 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:24:45,740 - db - INFO - Database initialization completed. +2024-10-29 12:24:45,752 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.011398 seconds, for an average of 0.00033523529411764704 seconds per hash. +2024-10-29 12:24:45,752 - utils - INFO - Checking models directory... +2024-10-29 12:24:45,752 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:24:45,753 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:24:45,753 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:24:45,753 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:24:45,753 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:24:45,753 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:24:45,753 - utils - INFO - Model downloads completed. +2024-10-29 12:24:45,760 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:24:45,762 - main - INFO - Application initialization complete +2024-10-29 12:25:09,382 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:25:09,382 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:25:09,435 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:25:09,439 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:25:09,439 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:25:09,475 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:25:38,622 - main - INFO - Application shutdown initiated +2024-10-29 12:25:41,456 - main - INFO - Starting application initialization +2024-10-29 12:25:41,464 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:25:41,469 - db - INFO - Database initialization completed. +2024-10-29 12:25:41,482 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.012197 seconds, for an average of 0.00035873529411764707 seconds per hash. +2024-10-29 12:25:41,482 - utils - INFO - Checking models directory... +2024-10-29 12:25:41,482 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:25:41,482 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:25:41,483 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:25:41,483 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:25:41,483 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:25:41,483 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:25:41,483 - utils - INFO - Model downloads completed. +2024-10-29 12:25:41,490 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:25:41,491 - main - INFO - Application initialization complete +2024-10-29 12:26:31,248 - main - INFO - Application shutdown initiated +2024-10-29 12:26:34,346 - main - INFO - Starting application initialization +2024-10-29 12:26:34,347 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:26:34,353 - db - INFO - Database initialization completed. +2024-10-29 12:26:34,366 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.013159 seconds, for an average of 0.0003870294117647059 seconds per hash. +2024-10-29 12:26:34,366 - utils - INFO - Checking models directory... +2024-10-29 12:26:34,366 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:26:34,366 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:26:34,367 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:26:34,367 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:26:34,367 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:26:34,367 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:26:34,367 - utils - INFO - Model downloads completed. +2024-10-29 12:26:34,374 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:26:34,375 - main - INFO - Application initialization complete +2024-10-29 12:26:41,354 - main - INFO - Application shutdown initiated +2024-10-29 12:26:44,080 - main - INFO - Starting application initialization +2024-10-29 12:26:44,081 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:26:44,085 - db - INFO - Database initialization completed. +2024-10-29 12:26:44,096 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.011666 seconds, for an average of 0.0003431176470588235 seconds per hash. +2024-10-29 12:26:44,097 - utils - INFO - Checking models directory... +2024-10-29 12:26:44,097 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:26:44,097 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:26:44,097 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:26:44,097 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:26:44,097 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:26:44,097 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:26:44,097 - utils - INFO - Model downloads completed. +2024-10-29 12:26:44,103 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:26:44,105 - main - INFO - Application initialization complete +2024-10-29 12:26:48,617 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:26:48,617 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:26:48,672 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:26:48,676 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:26:48,677 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:26:48,709 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:30:28,469 - main - INFO - Application shutdown initiated +2024-10-29 12:32:16,837 - main - INFO - Starting application initialization +2024-10-29 12:32:16,839 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:32:16,846 - db - INFO - Database initialization completed. +2024-10-29 12:32:16,859 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.012313 seconds, for an average of 0.0003621470588235294 seconds per hash. +2024-10-29 12:32:16,859 - utils - INFO - Checking models directory... +2024-10-29 12:32:16,859 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:32:16,860 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:32:16,860 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:32:16,860 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:32:16,860 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:32:16,860 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:32:16,860 - utils - INFO - Model downloads completed. +2024-10-29 12:32:16,868 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:32:16,870 - main - INFO - Application initialization complete +2024-10-29 12:32:20,368 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:32:20,369 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:32:20,403 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:32:20,407 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:32:20,407 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:32:20,437 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:32:22,429 - main - INFO - Retrieving content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:32:22,437 - main - INFO - Retrieved content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:34:45,278 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:34:45,279 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:34:45,348 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:34:45,351 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:34:45,351 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:34:45,381 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:34:47,713 - main - INFO - Retrieving content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:34:47,718 - main - INFO - Retrieved content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:34:52,878 - main - INFO - Processing advanced semantic search request for query: **Date:** March 22, 2024 + +**Patient:** Michael Davis, 50 years old + +**MRN:** 01234567 + +**Chief Complaint:** "I've been experiencing frequent urination and burning sensations while urinating" + +**History of Present Illness:** The patient is a 50-year-old who presents with a 6-month history of frequent urination, burning sensations while urinating, and occasional blood in the urine The patient reports working as a truck driver and has been experiencing increased stress due to long hours on the road and financial struggles **Past Medical History:** Prostate Cancer + +**Social History:** +The patient shares a small trailer with his wife and relies on public transportation He mentions feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 130/85, HR 76, Temp 98.6°F, Resp 16/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and prostate function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:34:52,879 - main - ERROR - Failed to enqueue job: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' +2024-10-29 12:34:52,879 - main - ERROR - Failed to process semantic search request: 500: Failed to queue semantic search: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' +2024-10-29 12:34:52,884 - main - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/main.py", line 544, in advanced_semantic_search + request.document_hash, + File "/Users/sidmohan/.pyenv/versions/3.10.13/lib/python3.10/site-packages/pydantic/main.py", line 856, in __getattr__ + raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') +AttributeError: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/main.py", line 587, in advanced_semantic_search + raise HTTPException( +fastapi.exceptions.HTTPException: 500: Failed to queue semantic search: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' + +2024-10-29 12:37:16,796 - main - INFO - Application shutdown initiated +2024-10-29 12:37:19,908 - main - INFO - Starting application initialization +2024-10-29 12:37:19,910 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:37:19,914 - db - INFO - Database initialization completed. +2024-10-29 12:37:19,925 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 34; Took 0.010929 seconds, for an average of 0.0003214411764705882 seconds per hash. +2024-10-29 12:37:19,925 - utils - INFO - Checking models directory... +2024-10-29 12:37:19,925 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:37:19,926 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:37:19,926 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Hermes-3-Llama-3.1-8B.Q4_K_M.gguf +2024-10-29 12:37:19,926 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/bge-m3-q8_0.gguf +2024-10-29 12:37:19,926 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/llava-llama-3-8b-v1_1-int4.gguf +2024-10-29 12:37:19,926 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf +2024-10-29 12:37:19,926 - utils - INFO - Model downloads completed. +2024-10-29 12:37:19,933 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:37:19,934 - main - INFO - Application initialization complete +2024-10-29 12:37:26,981 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:37:26,981 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:37:27,033 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:37:27,036 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:37:27,036 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:37:27,060 - main - INFO - Retrieved 10 stored documents from the database +2024-10-29 12:37:30,130 - main - INFO - Retrieving content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:37:30,136 - main - INFO - Retrieved content for document with document_hash: 47751b33e14a4f1fa93b76fd4f1a48c3ea3f3faf2eb8da595aef7dd2b7acd810 +2024-10-29 12:37:34,413 - main - INFO - Processing advanced semantic search request for query: **Date:** March 22, 2024 + +**Patient:** Michael Davis, 50 years old + +**MRN:** 01234567 + +**Chief Complaint:** "I've been experiencing frequent urination and burning sensations while urinating" + +**History of Present Illness:** The patient is a 50-year-old who presents with a 6-month history of frequent urination, burning sensations while urinating, and occasional blood in the urine The patient reports working as a truck driver and has been experiencing increased stress due to long hours on the road and financial struggles **Past Medical History:** Prostate Cancer + +**Social History:** +The patient shares a small trailer with his wife and relies on public transportation He mentions feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 130/85, HR 76, Temp 98.6°F, Resp 16/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and prostate function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:37:34,414 - main - ERROR - Failed to enqueue job: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' +2024-10-29 12:37:34,415 - main - ERROR - Failed to process semantic search request: 500: Failed to queue semantic search: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' +2024-10-29 12:37:34,419 - main - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/main.py", line 544, in advanced_semantic_search + request.document_hash, + File "/Users/sidmohan/.pyenv/versions/3.10.13/lib/python3.10/site-packages/pydantic/main.py", line 856, in __getattr__ + raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') +AttributeError: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/main.py", line 587, in advanced_semantic_search + raise HTTPException( +fastapi.exceptions.HTTPException: 500: Failed to queue semantic search: 'AdvancedSemanticSearchRequest' object has no attribute 'document_hash' + +2024-10-29 12:39:53,366 - main - INFO - Application shutdown initiated +2024-10-29 12:42:18,806 - main - INFO - Starting application initialization +2024-10-29 12:42:18,807 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:42:18,827 - db - INFO - Database initialization completed. +2024-10-29 12:42:18,837 - utils - INFO - Checking models directory... +2024-10-29 12:42:18,837 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:42:18,837 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:42:18,837 - utils - INFO - Model downloads completed. +2024-10-29 12:42:18,839 - main - INFO - Application initialization complete +2024-10-29 12:42:36,487 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:42:36,488 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:42:36,494 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:42:36,496 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:42:36,496 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:42:36,499 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:42:40,165 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:42:40,168 - main - INFO - Retrieved 0 semantic data types from the database +2024-10-29 12:42:40,168 - main - INFO - Processed 0 semantic data types +2024-10-29 12:42:40,172 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:42:40,185 - main - INFO - Retrieved 0 semantic data types from the database +2024-10-29 12:42:40,185 - main - INFO - Processed 0 semantic data types +2024-10-29 12:42:59,809 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'generic IDs...' +2024-10-29 12:42:59,810 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.5 seconds. +2024-10-29 12:42:59,810 - functions - INFO - That's an average of 466.64 ms per sentence and 2.143 sentences per second (and 0.0236 total characters per ms) using pooling method 'mean' +2024-10-29 12:42:59,834 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'Social Security Number Employee Number MRN...' +2024-10-29 12:42:59,834 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.0 seconds. +2024-10-29 12:42:59,834 - functions - INFO - That's an average of 19.26 ms per sentence and 51.935 sentences per second (and 2.1813 total characters per ms) using pooling method 'mean' +2024-10-29 12:43:02,245 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:02,245 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:02,249 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:43:02,251 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:02,251 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:02,253 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:43:02,926 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:02,926 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:02,931 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:43:02,933 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:02,933 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:02,937 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:43:13,702 - main - INFO - Processing document URL: None +2024-10-29 12:43:13,716 - main - INFO - Job enqueued successfully. Job ID: c0141355be7c361e70adc798528f04ee +2024-10-29 12:43:13,730 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:13,731 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:13,734 - main - INFO - Retrieved 0 stored documents from the database +2024-10-29 12:43:13,736 - rq.worker - INFO - file_uploads: worker.upload_file_task('/tmp/c0141355be7c361e70adc798528f04ee_note1.txt', None, None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'zip', None) (c0141355be7c361e70adc798528f04ee) +2024-10-29 12:43:14,220 - worker - INFO - Starting file upload task for file: /tmp/c0141355be7c361e70adc798528f04ee_note1.txt +2024-10-29 12:43:14,222 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:43:14,243 - db - INFO - Database initialization completed. +2024-10-29 12:43:14,249 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 2; Took 0.005255 seconds, for an average of 0.0026275 seconds per hash. +2024-10-29 12:43:14,250 - utils - INFO - Checking models directory... +2024-10-29 12:43:14,251 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:43:14,252 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:43:14,252 - utils - INFO - Model downloads completed. +2024-10-29 12:43:14,276 - worker - INFO - Successfully saved document and embedding with hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:43:14,279 - rq.worker - INFO - file_uploads: Job OK (c0141355be7c361e70adc798528f04ee) +2024-10-29 12:43:14,280 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:43:14,285 - rq.worker - INFO - Cleaning registries for queue: model_downloads +2024-10-29 12:43:14,286 - rq.worker - INFO - Cleaning registries for queue: file_uploads +2024-10-29 12:43:14,287 - rq.worker - INFO - Cleaning registries for queue: document_scans +2024-10-29 12:43:16,722 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:16,723 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:16,728 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:43:16,730 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:43:16,731 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:43:16,734 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:43:19,327 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:43:19,335 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:43:24,552 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:43:24,555 - main - INFO - Job enqueued successfully. Job ID: ba55ad865ef7c5cf16b31f4736a86893 +2024-10-29 12:43:24,555 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (ba55ad865ef7c5cf16b31f4736a86893) +2024-10-29 12:43:24,587 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:43:24,594 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:43:24,598 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 12:43:24,599 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:43:24,602 - functions - INFO - Received request for embedding for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' from IP address 'localhost' +2024-10-29 12:43:25,037 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 **Patient:** Emily Johnso...' +2024-10-29 12:43:25,038 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 12:43:25,039 - functions - INFO - That's an average of 393.57 ms per sentence and 2.541 sentences per second (and 4.4338 total characters per ms) using pooling method 'mean' +2024-10-29 12:43:25,040 - functions - INFO - Embedding calculated for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' in 0.00 seconds, or an average of 0.00 seconds per word. Now saving to database... +2024-10-29 12:43:25,040 - worker - ERROR - An error occurred while processing the request: 'NoneType' object has no attribute 'enqueue_write' +2024-10-29 12:43:25,044 - worker - ERROR - Error in scan task: 500: Internal Server Error +2024-10-29 12:43:25,045 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 311, in scan_document_task + embedding_response = await get_or_compute_embedding(embedding_request) + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 472, in get_or_compute_embedding + await db_writer.enqueue_write([embedding_instance]) # Enqueue the write operation using the db_writer instance directly +AttributeError: 'NoneType' object has no attribute 'enqueue_write' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 373, in scan_document_task + raise HTTPException(status_code=500, detail="Internal Server Error") +fastapi.exceptions.HTTPException: 500: Internal Server Error + +2024-10-29 12:43:25,049 - rq.worker - INFO - document_scans: Job OK (ba55ad865ef7c5cf16b31f4736a86893) +2024-10-29 12:43:25,049 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:44:33,795 - main - INFO - Application shutdown initiated +2024-10-29 12:44:36,666 - main - INFO - Starting application initialization +2024-10-29 12:44:36,668 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:44:36,673 - db - INFO - Database initialization completed. +2024-10-29 12:44:36,685 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011362 seconds, for an average of 0.0028405 seconds per hash. +2024-10-29 12:44:36,685 - utils - INFO - Checking models directory... +2024-10-29 12:44:36,685 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:44:36,685 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:44:36,685 - utils - INFO - Model downloads completed. +2024-10-29 12:44:36,688 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:44:36,689 - main - INFO - Application initialization complete +2024-10-29 12:44:46,426 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:44:46,426 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:44:46,433 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:44:46,435 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:44:46,435 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:44:46,437 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:44:48,446 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:44:48,453 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:44:52,484 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:44:52,488 - main - INFO - Job enqueued successfully. Job ID: a65666f6121137d3ff5d0004f64ad4c4 +2024-10-29 12:44:52,490 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (a65666f6121137d3ff5d0004f64ad4c4) +2024-10-29 12:44:52,528 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:44:52,530 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:44:52,537 - db - INFO - Database initialization completed. +2024-10-29 12:44:52,544 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005622 seconds, for an average of 0.0014055 seconds per hash. +2024-10-29 12:44:52,545 - utils - INFO - Checking models directory... +2024-10-29 12:44:52,545 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:44:52,546 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:44:52,546 - utils - INFO - Model downloads completed. +2024-10-29 12:44:52,547 - worker - ERROR - Error in scan task: name 'DatabaseWriter' is not defined +2024-10-29 12:44:52,552 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 301, in scan_document_task + db_writer = DatabaseWriter(queue) +NameError: name 'DatabaseWriter' is not defined + +2024-10-29 12:44:52,555 - rq.worker - INFO - document_scans: Job OK (a65666f6121137d3ff5d0004f64ad4c4) +2024-10-29 12:44:52,555 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:45:49,427 - main - INFO - Application shutdown initiated +2024-10-29 12:45:52,158 - main - INFO - Starting application initialization +2024-10-29 12:45:52,160 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:45:52,165 - db - INFO - Database initialization completed. +2024-10-29 12:45:52,177 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011145 seconds, for an average of 0.00278625 seconds per hash. +2024-10-29 12:45:52,177 - utils - INFO - Checking models directory... +2024-10-29 12:45:52,177 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:45:52,177 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:45:52,177 - utils - INFO - Model downloads completed. +2024-10-29 12:45:52,180 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:45:52,180 - main - INFO - Application initialization complete +2024-10-29 12:45:53,649 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:45:53,649 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:45:53,655 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:45:53,656 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:45:53,657 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:45:53,659 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:45:56,726 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:45:56,734 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:46:00,922 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:46:00,925 - main - INFO - Job enqueued successfully. Job ID: 319db1330d4cfc20d12b538f001ec830 +2024-10-29 12:46:00,928 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (319db1330d4cfc20d12b538f001ec830) +2024-10-29 12:46:01,007 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:46:01,008 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:46:01,017 - db - INFO - Database initialization completed. +2024-10-29 12:46:01,025 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.006847 seconds, for an average of 0.00171175 seconds per hash. +2024-10-29 12:46:01,025 - utils - INFO - Checking models directory... +2024-10-29 12:46:01,026 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:46:01,027 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:46:01,027 - utils - INFO - Model downloads completed. +2024-10-29 12:46:01,032 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004547 seconds, for an average of 0.00113675 seconds per hash. +2024-10-29 12:46:01,036 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:46:01,037 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 12:46:01,038 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:46:01,038 - worker - ERROR - An error occurred while processing the request: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' +2024-10-29 12:46:01,041 - worker - ERROR - Error in scan task: 500: Internal Server Error +2024-10-29 12:46:01,042 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 321, in scan_document_task + embedding_response = await get_or_compute_embedding(embedding_request, db_writer=db_writer) # Pass db_writer here +TypeError: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 383, in scan_document_task + raise HTTPException(status_code=500, detail="Internal Server Error") +fastapi.exceptions.HTTPException: 500: Internal Server Error + +2024-10-29 12:46:01,058 - rq.worker - INFO - document_scans: Job OK (319db1330d4cfc20d12b538f001ec830) +2024-10-29 12:46:01,059 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:48:14,006 - main - INFO - Application shutdown initiated +2024-10-29 12:48:16,892 - main - INFO - Starting application initialization +2024-10-29 12:48:16,893 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:48:16,900 - db - INFO - Database initialization completed. +2024-10-29 12:48:16,911 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011761 seconds, for an average of 0.00294025 seconds per hash. +2024-10-29 12:48:16,912 - utils - INFO - Checking models directory... +2024-10-29 12:48:16,912 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:48:16,912 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:48:16,912 - utils - INFO - Model downloads completed. +2024-10-29 12:48:16,915 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:48:16,916 - main - INFO - Application initialization complete +2024-10-29 12:48:18,337 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:48:18,337 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:48:18,346 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:48:18,352 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:48:18,352 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:48:18,356 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:48:20,419 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:48:20,425 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:48:27,808 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:48:27,811 - main - INFO - Job enqueued successfully. Job ID: 67610bbe4bec6d7374fe74a20d2d95a1 +2024-10-29 12:48:27,813 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (67610bbe4bec6d7374fe74a20d2d95a1) +2024-10-29 12:48:27,847 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:48:27,849 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:48:27,857 - db - INFO - Database initialization completed. +2024-10-29 12:48:27,866 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.008943 seconds, for an average of 0.00223575 seconds per hash. +2024-10-29 12:48:27,867 - utils - INFO - Checking models directory... +2024-10-29 12:48:27,867 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:48:27,868 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:48:27,868 - utils - INFO - Model downloads completed. +2024-10-29 12:48:27,872 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.00336 seconds, for an average of 0.00084 seconds per hash. +2024-10-29 12:48:27,876 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:48:27,877 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 12:48:27,878 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:48:27,878 - worker - ERROR - An error occurred while processing the request: 'DatabaseWriter' object has no attribute 'client' +2024-10-29 12:48:27,881 - worker - ERROR - Error in scan task: 500: Internal Server Error +2024-10-29 12:48:27,882 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 321, in scan_document_task + embedding_response = await get_or_compute_embedding(embedding_request, db_writer) # Pass db_writer here + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 418, in get_or_compute_embedding + client_ip: str = None, +AttributeError: 'DatabaseWriter' object has no attribute 'client' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 383, in scan_document_task + raise HTTPException(status_code=500, detail="Internal Server Error") +fastapi.exceptions.HTTPException: 500: Internal Server Error + +2024-10-29 12:48:27,884 - rq.worker - INFO - document_scans: Job OK (67610bbe4bec6d7374fe74a20d2d95a1) +2024-10-29 12:48:27,885 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:48:57,442 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:48:57,442 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:48:57,448 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:48:57,474 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 12:48:57,480 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 12:48:57,480 - main - INFO - Processed 1 semantic data types +2024-10-29 12:49:02,688 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:49:02,689 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:49:02,716 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:49:02,718 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:49:02,718 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:49:02,721 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:49:04,430 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:49:04,439 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:49:09,892 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:49:09,895 - main - INFO - Job enqueued successfully. Job ID: beef1b0053b0191b310cf816f9918da1 +2024-10-29 12:49:09,898 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (beef1b0053b0191b310cf816f9918da1) +2024-10-29 12:49:09,937 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:49:09,938 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:49:09,955 - db - INFO - Database initialization completed. +2024-10-29 12:49:09,962 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.006296 seconds, for an average of 0.001574 seconds per hash. +2024-10-29 12:49:09,963 - utils - INFO - Checking models directory... +2024-10-29 12:49:09,964 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:49:09,965 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:49:09,965 - utils - INFO - Model downloads completed. +2024-10-29 12:49:09,970 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004232 seconds, for an average of 0.001058 seconds per hash. +2024-10-29 12:49:09,973 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:49:09,975 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 12:49:09,975 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:49:09,976 - worker - ERROR - An error occurred while processing the request: 'DatabaseWriter' object has no attribute 'client' +2024-10-29 12:49:09,979 - worker - ERROR - Error in scan task: 500: Internal Server Error +2024-10-29 12:49:09,980 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 321, in scan_document_task + embedding_response = await get_or_compute_embedding(embedding_request, db_writer) # Pass db_writer here + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 418, in get_or_compute_embedding + client_ip: str = None, +AttributeError: 'DatabaseWriter' object has no attribute 'client' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 383, in scan_document_task + raise HTTPException(status_code=500, detail="Internal Server Error") +fastapi.exceptions.HTTPException: 500: Internal Server Error + +2024-10-29 12:49:09,983 - rq.worker - INFO - document_scans: Job OK (beef1b0053b0191b310cf816f9918da1) +2024-10-29 12:49:09,983 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:51:38,919 - main - INFO - Application shutdown initiated +2024-10-29 12:51:41,618 - main - INFO - Starting application initialization +2024-10-29 12:51:41,619 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:51:41,624 - db - INFO - Database initialization completed. +2024-10-29 12:51:41,636 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.012055 seconds, for an average of 0.00301375 seconds per hash. +2024-10-29 12:51:41,637 - utils - INFO - Checking models directory... +2024-10-29 12:51:41,637 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:51:41,637 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:51:41,637 - utils - INFO - Model downloads completed. +2024-10-29 12:51:41,639 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:51:41,640 - main - INFO - Application initialization complete +2024-10-29 12:51:46,873 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:51:46,873 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:51:46,882 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:51:46,884 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:51:46,884 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:51:46,887 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:51:49,034 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:51:49,041 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:51:54,227 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:51:54,230 - main - INFO - Job enqueued successfully. Job ID: 6349a34cd0986243ff23b4ba1fcf4cbd +2024-10-29 12:51:54,232 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (6349a34cd0986243ff23b4ba1fcf4cbd) +2024-10-29 12:51:54,267 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:51:54,269 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:51:54,277 - db - INFO - Database initialization completed. +2024-10-29 12:51:54,283 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005551 seconds, for an average of 0.00138775 seconds per hash. +2024-10-29 12:51:54,284 - utils - INFO - Checking models directory... +2024-10-29 12:51:54,285 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:51:54,286 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:51:54,286 - utils - INFO - Model downloads completed. +2024-10-29 12:51:54,291 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003778 seconds, for an average of 0.0009445 seconds per hash. +2024-10-29 12:51:54,294 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:51:54,295 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 12:51:54,296 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:51:54,296 - worker - ERROR - An error occurred while processing the request: 'DatabaseWriter' object has no attribute 'client' +2024-10-29 12:51:54,299 - worker - ERROR - Error in scan task: 500: Internal Server Error +2024-10-29 12:51:54,300 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 321, in scan_document_task + embedding_response = await get_or_compute_embedding(embedding_request, db_writer) # Pass db_writer here + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 418, in get_or_compute_embedding + client_ip: str = None, +AttributeError: 'DatabaseWriter' object has no attribute 'client' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 383, in scan_document_task + raise HTTPException(status_code=500, detail="Internal Server Error") +fastapi.exceptions.HTTPException: 500: Internal Server Error + +2024-10-29 12:51:54,303 - rq.worker - INFO - document_scans: Job OK (6349a34cd0986243ff23b4ba1fcf4cbd) +2024-10-29 12:51:54,304 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:53:54,726 - main - INFO - Application shutdown initiated +2024-10-29 12:53:57,997 - main - INFO - Starting application initialization +2024-10-29 12:53:57,999 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:53:58,003 - db - INFO - Database initialization completed. +2024-10-29 12:53:58,015 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011243 seconds, for an average of 0.00281075 seconds per hash. +2024-10-29 12:53:58,015 - utils - INFO - Checking models directory... +2024-10-29 12:53:58,015 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:53:58,015 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:53:58,015 - utils - INFO - Model downloads completed. +2024-10-29 12:53:58,018 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:53:58,018 - main - INFO - Application initialization complete +2024-10-29 12:53:58,019 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:53:58,019 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:53:58,041 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:53:58,042 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:53:58,042 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:53:58,045 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:54:01,951 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:54:01,962 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:54:05,709 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:54:05,712 - main - INFO - Job enqueued successfully. Job ID: 8036a0c47e51d07e699751b824b1d0af +2024-10-29 12:54:05,714 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (8036a0c47e51d07e699751b824b1d0af) +2024-10-29 12:54:05,770 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:54:05,772 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:54:05,781 - db - INFO - Database initialization completed. +2024-10-29 12:54:05,788 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.006946 seconds, for an average of 0.0017365 seconds per hash. +2024-10-29 12:54:05,789 - utils - INFO - Checking models directory... +2024-10-29 12:54:05,789 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:54:05,790 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:54:05,790 - utils - INFO - Model downloads completed. +2024-10-29 12:54:05,795 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003714 seconds, for an average of 0.0009285 seconds per hash. +2024-10-29 12:54:06,193 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 12:54:06,194 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 12:54:06,195 - functions - INFO - That's an average of 358.99 ms per sentence and 2.786 sentences per second (and 4.8804 total characters per ms) using pooling method 'mean' +2024-10-29 12:54:06,196 - worker - ERROR - Error in scan task: 'NoneType' object has no attribute 'enqueue_write' +2024-10-29 12:54:06,198 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 313, in scan_document_task + embedding_response = await get_or_compute_embedding( + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 472, in get_or_compute_embedding + ) +AttributeError: 'NoneType' object has no attribute 'enqueue_write' + +2024-10-29 12:54:06,201 - rq.worker - INFO - document_scans: Job OK (8036a0c47e51d07e699751b824b1d0af) +2024-10-29 12:54:06,202 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:54:48,925 - main - INFO - Application shutdown initiated +2024-10-29 12:54:52,310 - main - INFO - Starting application initialization +2024-10-29 12:54:52,312 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:54:52,319 - db - INFO - Database initialization completed. +2024-10-29 12:54:52,334 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.014705 seconds, for an average of 0.00367625 seconds per hash. +2024-10-29 12:54:52,335 - utils - INFO - Checking models directory... +2024-10-29 12:54:52,335 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:54:52,335 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:54:52,335 - utils - INFO - Model downloads completed. +2024-10-29 12:54:52,338 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:54:52,339 - main - INFO - Application initialization complete +2024-10-29 12:54:53,107 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:54:53,107 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:54:53,115 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:54:53,116 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:54:53,116 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:54:53,119 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:54:55,074 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:54:55,078 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:54:59,594 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:54:59,597 - main - INFO - Job enqueued successfully. Job ID: 7da71c847b859b74c4a9a492a89d0aa8 +2024-10-29 12:54:59,599 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (7da71c847b859b74c4a9a492a89d0aa8) +2024-10-29 12:54:59,637 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:54:59,638 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:54:59,646 - db - INFO - Database initialization completed. +2024-10-29 12:54:59,653 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005632 seconds, for an average of 0.001408 seconds per hash. +2024-10-29 12:54:59,653 - utils - INFO - Checking models directory... +2024-10-29 12:54:59,654 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:54:59,655 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:54:59,655 - utils - INFO - Model downloads completed. +2024-10-29 12:54:59,660 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004239 seconds, for an average of 0.00105975 seconds per hash. +2024-10-29 12:54:59,660 - worker - ERROR - Error in scan task: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' +2024-10-29 12:54:59,662 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 313, in scan_document_task + embedding_response = await get_or_compute_embedding( +TypeError: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' + +2024-10-29 12:54:59,665 - rq.worker - INFO - document_scans: Job OK (7da71c847b859b74c4a9a492a89d0aa8) +2024-10-29 12:54:59,666 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:55:24,766 - main - INFO - Application shutdown initiated +2024-10-29 12:55:27,237 - main - INFO - Starting application initialization +2024-10-29 12:55:27,239 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:55:27,243 - db - INFO - Database initialization completed. +2024-10-29 12:55:27,255 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011628 seconds, for an average of 0.002907 seconds per hash. +2024-10-29 12:55:27,256 - utils - INFO - Checking models directory... +2024-10-29 12:55:27,256 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:55:27,256 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:55:27,256 - utils - INFO - Model downloads completed. +2024-10-29 12:55:27,258 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:55:27,259 - main - INFO - Application initialization complete +2024-10-29 12:55:32,372 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:55:32,372 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:55:32,379 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:55:32,381 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:55:32,381 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:55:32,384 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:55:34,527 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:55:34,535 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:55:38,632 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:55:38,635 - main - INFO - Job enqueued successfully. Job ID: f39e61d51584995e465b305eb55ce74b +2024-10-29 12:55:38,637 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (f39e61d51584995e465b305eb55ce74b) +2024-10-29 12:55:38,669 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:55:38,670 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:55:38,677 - db - INFO - Database initialization completed. +2024-10-29 12:55:38,683 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005698 seconds, for an average of 0.0014245 seconds per hash. +2024-10-29 12:55:38,684 - utils - INFO - Checking models directory... +2024-10-29 12:55:38,684 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:55:38,685 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:55:38,685 - utils - INFO - Model downloads completed. +2024-10-29 12:55:38,689 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003456 seconds, for an average of 0.000864 seconds per hash. +2024-10-29 12:55:38,690 - worker - ERROR - Error in scan task: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' +2024-10-29 12:55:38,692 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 313, in scan_document_task + embedding_response = await get_or_compute_embedding( +TypeError: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' + +2024-10-29 12:55:38,707 - rq.worker - INFO - document_scans: Job OK (f39e61d51584995e465b305eb55ce74b) +2024-10-29 12:55:38,707 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:55:54,393 - main - INFO - Application shutdown initiated +2024-10-29 12:55:57,150 - main - INFO - Starting application initialization +2024-10-29 12:55:57,151 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:55:57,156 - db - INFO - Database initialization completed. +2024-10-29 12:55:57,167 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.010906 seconds, for an average of 0.0027265 seconds per hash. +2024-10-29 12:55:57,167 - utils - INFO - Checking models directory... +2024-10-29 12:55:57,167 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:55:57,167 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:55:57,167 - utils - INFO - Model downloads completed. +2024-10-29 12:55:57,169 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:55:57,170 - main - INFO - Application initialization complete +2024-10-29 12:56:26,834 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:56:26,834 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:56:26,850 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:56:26,853 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:56:26,853 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:56:26,856 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:56:29,032 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:56:29,039 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:56:33,301 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:56:33,305 - main - INFO - Job enqueued successfully. Job ID: 3d4b65bac5c52cb376bb427f66ac8601 +2024-10-29 12:56:33,307 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (3d4b65bac5c52cb376bb427f66ac8601) +2024-10-29 12:56:33,341 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:56:33,342 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:56:33,353 - db - INFO - Database initialization completed. +2024-10-29 12:56:33,360 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.006657 seconds, for an average of 0.00166425 seconds per hash. +2024-10-29 12:56:33,361 - utils - INFO - Checking models directory... +2024-10-29 12:56:33,362 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:56:33,363 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:56:33,363 - utils - INFO - Model downloads completed. +2024-10-29 12:56:33,370 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.00577 seconds, for an average of 0.0014425 seconds per hash. +2024-10-29 12:56:33,812 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 12:56:33,814 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 12:56:33,814 - functions - INFO - That's an average of 408.12 ms per sentence and 2.450 sentences per second (and 4.2928 total characters per ms) using pooling method 'mean' +2024-10-29 12:56:33,815 - worker - ERROR - Error in scan task: 'NoneType' object has no attribute 'enqueue_write' +2024-10-29 12:56:33,817 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 313, in scan_document_task + embedding_response = await get_or_compute_embedding( + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 472, in get_or_compute_embedding + ) +AttributeError: 'NoneType' object has no attribute 'enqueue_write' + +2024-10-29 12:56:33,830 - rq.worker - INFO - document_scans: Job OK (3d4b65bac5c52cb376bb427f66ac8601) +2024-10-29 12:56:33,830 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 12:57:59,686 - main - INFO - Application shutdown initiated +2024-10-29 12:58:03,048 - main - INFO - Starting application initialization +2024-10-29 12:58:03,049 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:58:03,054 - db - INFO - Database initialization completed. +2024-10-29 12:58:03,066 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011537 seconds, for an average of 0.00288425 seconds per hash. +2024-10-29 12:58:03,066 - utils - INFO - Checking models directory... +2024-10-29 12:58:03,066 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:58:03,067 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:58:03,067 - utils - INFO - Model downloads completed. +2024-10-29 12:58:03,069 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 12:58:03,070 - main - INFO - Application initialization complete +2024-10-29 12:58:03,071 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:58:03,071 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:58:03,079 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:58:03,080 - main - INFO - Received request to retrieve all stored documents +2024-10-29 12:58:03,080 - main - INFO - Retrieving all stored documents from the database +2024-10-29 12:58:03,082 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 12:58:04,766 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:58:04,774 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 12:58:09,100 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 12:58:09,103 - main - INFO - Job enqueued successfully. Job ID: f5874b858fd28ffc75f582951a2e6f4e +2024-10-29 12:58:09,104 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (f5874b858fd28ffc75f582951a2e6f4e) +2024-10-29 12:58:09,141 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 12:58:09,143 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 12:58:09,150 - db - INFO - Database initialization completed. +2024-10-29 12:58:09,156 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005194 seconds, for an average of 0.0012985 seconds per hash. +2024-10-29 12:58:09,157 - utils - INFO - Checking models directory... +2024-10-29 12:58:09,157 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 12:58:09,158 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 12:58:09,159 - utils - INFO - Model downloads completed. +2024-10-29 12:58:09,162 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003348 seconds, for an average of 0.000837 seconds per hash. +2024-10-29 12:58:09,703 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 12:58:09,704 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 12:58:09,705 - functions - INFO - That's an average of 394.72 ms per sentence and 2.533 sentences per second (and 4.4386 total characters per ms) using pooling method 'mean' +2024-10-29 12:58:09,706 - worker - ERROR - Error in scan task: 'NoneType' object has no attribute 'enqueue_write' +2024-10-29 12:58:09,709 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 316, in scan_document_task + embedding_response = await get_or_compute_embedding( + File "/Users/sidmohan/Desktop/codexify/backend/src/functions.py", line 472, in get_or_compute_embedding + ) +AttributeError: 'NoneType' object has no attribute 'enqueue_write' + +2024-10-29 12:58:09,724 - rq.worker - INFO - document_scans: Job OK (f5874b858fd28ffc75f582951a2e6f4e) +2024-10-29 12:58:09,724 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 13:01:38,137 - main - INFO - Application shutdown initiated +2024-10-29 13:01:41,005 - main - INFO - Starting application initialization +2024-10-29 13:01:41,006 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:01:41,010 - db - INFO - Database initialization completed. +2024-10-29 13:01:41,021 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.010796 seconds, for an average of 0.002699 seconds per hash. +2024-10-29 13:01:41,021 - utils - INFO - Checking models directory... +2024-10-29 13:01:41,021 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:01:41,022 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:01:41,022 - utils - INFO - Model downloads completed. +2024-10-29 13:01:41,024 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:01:41,025 - main - INFO - Application initialization complete +2024-10-29 13:01:49,533 - main - INFO - Application shutdown initiated +2024-10-29 13:01:52,138 - main - INFO - Starting application initialization +2024-10-29 13:01:52,139 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:01:52,144 - db - INFO - Database initialization completed. +2024-10-29 13:01:52,155 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011238 seconds, for an average of 0.0028095 seconds per hash. +2024-10-29 13:01:52,155 - utils - INFO - Checking models directory... +2024-10-29 13:01:52,155 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:01:52,156 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:01:52,156 - utils - INFO - Model downloads completed. +2024-10-29 13:01:52,158 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:01:52,158 - main - INFO - Application initialization complete +2024-10-29 13:02:03,483 - main - INFO - Application shutdown initiated +2024-10-29 13:02:06,109 - main - INFO - Starting application initialization +2024-10-29 13:02:06,110 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:02:06,115 - db - INFO - Database initialization completed. +2024-10-29 13:02:06,128 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.012814 seconds, for an average of 0.0032035 seconds per hash. +2024-10-29 13:02:06,128 - utils - INFO - Checking models directory... +2024-10-29 13:02:06,128 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:02:06,129 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:02:06,129 - utils - INFO - Model downloads completed. +2024-10-29 13:02:06,131 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:02:06,132 - main - INFO - Application initialization complete +2024-10-29 13:02:30,704 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:02:30,705 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:02:30,713 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:02:30,715 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:02:30,716 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:02:30,718 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:02:32,504 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:02:32,512 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:02:36,743 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:02:36,747 - main - INFO - Job enqueued successfully. Job ID: 0e78cf82f2cbd39d970a38420d5e94c8 +2024-10-29 13:02:36,749 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (0e78cf82f2cbd39d970a38420d5e94c8) +2024-10-29 13:02:36,810 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 13:02:36,811 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:02:36,819 - db - INFO - Database initialization completed. +2024-10-29 13:02:36,825 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005589 seconds, for an average of 0.00139725 seconds per hash. +2024-10-29 13:02:36,826 - utils - INFO - Checking models directory... +2024-10-29 13:02:36,827 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:02:36,828 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:02:36,828 - utils - INFO - Model downloads completed. +2024-10-29 13:02:36,833 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004259 seconds, for an average of 0.00106475 seconds per hash. +2024-10-29 13:02:36,937 - worker - ERROR - Error in scan task: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' +2024-10-29 13:02:36,941 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 316, in scan_document_task + embedding_response = await get_or_compute_embedding( +TypeError: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' + +2024-10-29 13:02:36,965 - rq.worker - INFO - document_scans: Job OK (0e78cf82f2cbd39d970a38420d5e94c8) +2024-10-29 13:02:36,966 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 13:02:54,315 - main - INFO - Application shutdown initiated +2024-10-29 13:02:58,924 - main - INFO - Starting application initialization +2024-10-29 13:02:58,949 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:02:58,954 - db - INFO - Database initialization completed. +2024-10-29 13:02:58,965 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011231 seconds, for an average of 0.00280775 seconds per hash. +2024-10-29 13:02:58,965 - utils - INFO - Checking models directory... +2024-10-29 13:02:58,965 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:02:58,966 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:02:58,966 - utils - INFO - Model downloads completed. +2024-10-29 13:02:58,968 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:02:58,968 - main - INFO - Application initialization complete +2024-10-29 13:03:10,759 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:03:10,759 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:03:10,766 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:03:10,792 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 13:03:10,795 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 13:03:10,795 - main - INFO - Processed 1 semantic data types +2024-10-29 13:03:13,039 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:03:13,040 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:03:13,044 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:03:13,046 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:03:13,046 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:03:13,048 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:03:16,355 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:03:16,361 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:03:23,680 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:03:23,684 - main - INFO - Job enqueued successfully. Job ID: e9edafd83e937cd31062b221ea7e8393 +2024-10-29 13:03:23,684 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (e9edafd83e937cd31062b221ea7e8393) +2024-10-29 13:03:23,716 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 13:03:23,717 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:03:23,727 - db - INFO - Database initialization completed. +2024-10-29 13:03:23,732 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004436 seconds, for an average of 0.001109 seconds per hash. +2024-10-29 13:03:23,732 - utils - INFO - Checking models directory... +2024-10-29 13:03:23,733 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:03:23,733 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:03:23,734 - utils - INFO - Model downloads completed. +2024-10-29 13:03:23,738 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003477 seconds, for an average of 0.00086925 seconds per hash. +2024-10-29 13:03:23,840 - worker - ERROR - Error in scan task: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' +2024-10-29 13:03:23,842 - worker - ERROR - Traceback (most recent call last): + File "/Users/sidmohan/Desktop/codexify/backend/src/worker.py", line 316, in scan_document_task + embedding_response = await get_or_compute_embedding( +TypeError: get_or_compute_embedding() got an unexpected keyword argument 'db_writer' + +2024-10-29 13:03:23,845 - rq.worker - INFO - document_scans: Job OK (e9edafd83e937cd31062b221ea7e8393) +2024-10-29 13:03:23,845 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 13:07:19,517 - main - INFO - Application shutdown initiated +2024-10-29 13:07:22,090 - main - INFO - Starting application initialization +2024-10-29 13:07:22,091 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:07:22,095 - db - INFO - Database initialization completed. +2024-10-29 13:07:22,107 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011494 seconds, for an average of 0.0028735 seconds per hash. +2024-10-29 13:07:22,107 - utils - INFO - Checking models directory... +2024-10-29 13:07:22,107 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:07:22,107 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:07:22,107 - utils - INFO - Model downloads completed. +2024-10-29 13:07:22,110 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:07:22,110 - main - INFO - Application initialization complete +2024-10-29 13:07:41,609 - rq.worker - INFO - Warm shut down requested +2024-10-29 13:07:41,609 - __main__ - INFO - Worker stopped by user +2024-10-29 13:07:42,062 - rq.scheduler - INFO - Scheduler stopping, releasing locks for document_scans,file_uploads,model_downloads... +2024-10-29 13:07:42,072 - rq.scheduler - INFO - Scheduler with PID 90244 has stopped +2024-10-29 13:07:42,363 - rq.worker - INFO - Unsubscribing from channel rq:pubsub:worker-90242 +2024-10-29 13:07:45,374 - __main__ - INFO - Initializing worker... +2024-10-29 13:07:45,374 - __main__ - INFO - Worker listening to queues: ['model_downloads', 'file_uploads', 'document_scans'] +2024-10-29 13:07:45,383 - __main__ - INFO - Worker started successfully with PID: 96986 +2024-10-29 13:07:46,793 - __mp_main__ - INFO - Connecting to Redis at localhost:6379 +2024-10-29 13:07:46,795 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:07:46,800 - db - INFO - Database initialization completed. +2024-10-29 13:07:46,812 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011438 seconds, for an average of 0.0028595 seconds per hash. +2024-10-29 13:07:46,812 - utils - INFO - Checking models directory... +2024-10-29 13:07:46,812 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:07:46,812 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:07:46,812 - utils - INFO - Model downloads completed. +2024-10-29 13:07:46,815 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:07:46,819 - rq.worker - INFO - Worker rq:worker:worker-96986: started, version 1.10.1 +2024-10-29 13:07:46,819 - rq.worker - INFO - Subscribing to channel rq:pubsub:worker-96986 +2024-10-29 13:07:46,821 - rq.worker - INFO - *** Listening on model_downloads, file_uploads, document_scans... +2024-10-29 13:07:46,821 - rq.scheduler - INFO - Trying to acquire locks for model_downloads, file_uploads, document_scans +2024-10-29 13:07:46,828 - rq.worker - INFO - Cleaning registries for queue: model_downloads +2024-10-29 13:07:46,829 - rq.worker - INFO - Cleaning registries for queue: file_uploads +2024-10-29 13:07:46,830 - rq.worker - INFO - Cleaning registries for queue: document_scans +2024-10-29 13:07:48,158 - rq.scheduler - INFO - Scheduler for file_uploads,document_scans,model_downloads started with PID 96991 +2024-10-29 13:08:36,767 - main - INFO - Application shutdown initiated +2024-10-29 13:08:39,552 - main - INFO - Starting application initialization +2024-10-29 13:08:39,554 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:08:39,560 - db - INFO - Database initialization completed. +2024-10-29 13:08:39,573 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.012719 seconds, for an average of 0.00317975 seconds per hash. +2024-10-29 13:08:39,573 - utils - INFO - Checking models directory... +2024-10-29 13:08:39,573 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:08:39,574 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:08:39,574 - utils - INFO - Model downloads completed. +2024-10-29 13:08:39,576 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:08:39,577 - main - INFO - Application initialization complete +2024-10-29 13:08:40,285 - main - INFO - Application shutdown initiated +2024-10-29 13:08:42,727 - main - INFO - Starting application initialization +2024-10-29 13:08:42,728 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:08:42,734 - db - INFO - Database initialization completed. +2024-10-29 13:08:42,746 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.011061 seconds, for an average of 0.00276525 seconds per hash. +2024-10-29 13:08:42,746 - utils - INFO - Checking models directory... +2024-10-29 13:08:42,746 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:08:42,746 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:08:42,746 - utils - INFO - Model downloads completed. +2024-10-29 13:08:42,748 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:08:42,748 - main - INFO - Application initialization complete +2024-10-29 13:08:44,471 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:08:44,471 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:08:44,477 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:08:44,479 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:08:44,479 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:08:44,481 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:08:47,242 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:08:47,249 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:08:52,009 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:08:52,013 - main - INFO - Job enqueued successfully. Job ID: 6f266e3eccc6b1e4159f783cbbd1a498 +2024-10-29 13:08:52,015 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (6f266e3eccc6b1e4159f783cbbd1a498) +2024-10-29 13:08:52,055 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 13:08:52,057 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:08:52,063 - db - INFO - Database initialization completed. +2024-10-29 13:08:52,069 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.00524 seconds, for an average of 0.00131 seconds per hash. +2024-10-29 13:08:52,070 - utils - INFO - Checking models directory... +2024-10-29 13:08:52,070 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:08:52,071 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:08:52,071 - utils - INFO - Model downloads completed. +2024-10-29 13:08:52,076 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004331 seconds, for an average of 0.00108275 seconds per hash. +2024-10-29 13:08:52,599 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 13:08:52,601 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 13:08:52,601 - functions - INFO - That's an average of 383.32 ms per sentence and 2.609 sentences per second (and 4.5706 total characters per ms) using pooling method 'mean' +2024-10-29 13:08:52,605 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:08:52,607 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 13:08:52,607 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:08:52,607 - functions - INFO - Received request for embedding for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' from IP address 'None' +2024-10-29 13:08:52,811 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 **Patient:** Emily Johnso...' +2024-10-29 13:08:52,812 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.2 seconds. +2024-10-29 13:08:52,813 - functions - INFO - That's an average of 202.59 ms per sentence and 4.936 sentences per second (and 8.6136 total characters per ms) using pooling method 'mean' +2024-10-29 13:08:52,813 - functions - INFO - Embedding calculated for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' in 0.00 seconds, or an average of 0.00 seconds per word. Now saving to database... +2024-10-29 13:08:52,814 - worker - INFO - Computed embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:08:52,827 - worker - INFO - Finished advanced search in 0.225305 seconds. Found 0 results. +2024-10-29 13:08:52,830 - rq.worker - INFO - document_scans: Job OK (6f266e3eccc6b1e4159f783cbbd1a498) +2024-10-29 13:08:52,830 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 13:09:39,327 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 13:09:39,330 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 13:09:39,330 - main - INFO - Processed 1 semantic data types +2024-10-29 13:09:39,332 - main - INFO - Retrieving all semantic data types from the database +2024-10-29 13:09:39,333 - main - INFO - Retrieved 1 semantic data types from the database +2024-10-29 13:09:39,333 - main - INFO - Processed 1 semantic data types +2024-10-29 13:09:48,031 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'back pain...' +2024-10-29 13:09:48,032 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.4 seconds. +2024-10-29 13:09:48,032 - functions - INFO - That's an average of 441.15 ms per sentence and 2.267 sentences per second (and 0.0204 total characters per ms) using pooling method 'mean' +2024-10-29 13:09:48,048 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text 'back pain...' +2024-10-29 13:09:48,049 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.0 seconds. +2024-10-29 13:09:48,049 - functions - INFO - That's an average of 12.54 ms per sentence and 79.751 sentences per second (and 0.7178 total characters per ms) using pooling method 'mean' +2024-10-29 13:09:50,180 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:09:50,180 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:09:50,190 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:09:50,195 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:09:50,195 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:09:50,198 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:09:51,764 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:09:51,768 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:09:57,352 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:09:57,354 - main - INFO - Job enqueued successfully. Job ID: 4a98a0d97d31925137184c4dbdd23be4 +2024-10-29 13:09:57,357 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', 'allergy', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (4a98a0d97d31925137184c4dbdd23be4) +2024-10-29 13:09:57,407 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 13:09:57,409 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:09:57,416 - db - INFO - Database initialization completed. +2024-10-29 13:09:57,422 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005152 seconds, for an average of 0.001288 seconds per hash. +2024-10-29 13:09:57,423 - utils - INFO - Checking models directory... +2024-10-29 13:09:57,423 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:09:57,424 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:09:57,424 - utils - INFO - Model downloads completed. +2024-10-29 13:09:57,429 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.003988 seconds, for an average of 0.000997 seconds per hash. +2024-10-29 13:09:57,843 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 13:09:57,844 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.3 seconds. +2024-10-29 13:09:57,845 - functions - INFO - That's an average of 273.32 ms per sentence and 3.659 sentences per second (and 6.4100 total characters per ms) using pooling method 'mean' +2024-10-29 13:09:57,850 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:09:57,851 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 13:09:57,852 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:09:57,852 - functions - INFO - Received request for embedding for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' from IP address 'None' +2024-10-29 13:09:58,032 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 **Patient:** Emily Johnso...' +2024-10-29 13:09:58,033 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.2 seconds. +2024-10-29 13:09:58,033 - functions - INFO - That's an average of 178.57 ms per sentence and 5.600 sentences per second (and 9.7719 total characters per ms) using pooling method 'mean' +2024-10-29 13:09:58,034 - functions - INFO - Embedding calculated for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' in 0.00 seconds, or an average of 0.00 seconds per word. Now saving to database... +2024-10-29 13:09:58,035 - worker - INFO - Computed embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:09:58,041 - worker - INFO - Finished advanced search in 0.195472 seconds. Found 0 results. +2024-10-29 13:09:58,045 - rq.worker - INFO - document_scans: Job OK (4a98a0d97d31925137184c4dbdd23be4) +2024-10-29 13:09:58,045 - rq.worker - INFO - Result is kept for 86400 seconds +2024-10-29 13:10:13,947 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:10:13,947 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:10:13,958 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:10:13,959 - main - INFO - Received request to retrieve all stored documents +2024-10-29 13:10:13,960 - main - INFO - Retrieving all stored documents from the database +2024-10-29 13:10:13,962 - main - INFO - Retrieved 1 stored documents from the database +2024-10-29 13:10:16,915 - main - INFO - Retrieving content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:10:16,918 - main - INFO - Retrieved content for document with document_hash: d1b1eea2129f4a35fdc50ff11326e9ee580cfa350eba01b822579c02b1f0c38f +2024-10-29 13:13:49,718 - main - INFO - Processing advanced semantic search request for query: **Date:** April 10, 2024 + +**Patient:** Emily Johnson, 35 years old + +**MRN:** 00987654 + +**Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" + +**History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism + +**Social History:** +The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** +- General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6°F, Resp 14/min + +**Assessment/Plan:** +- Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:13:49,722 - main - INFO - Job enqueued successfully. Job ID: c802d85a865e070edc18f7147c003a1f +2024-10-29 13:13:49,724 - rq.worker - INFO - document_scans: worker.scan_document_task(None, 'nomic-embed-text-v1.5.Q6_K', 'mean', '', 'records', 'json', '**Date:** April 10, 2024\n\n**Patient:** Emily Johnson, 35 years old\n\n**..., 0.01, 10, 'spearman_rho') (c802d85a865e070edc18f7147c003a1f) +2024-10-29 13:13:49,758 - worker - INFO - Starting document scan task for document hash: None +2024-10-29 13:13:49,760 - db - INFO - Initializing database, creating tables, and setting SQLite PRAGMAs... +2024-10-29 13:13:49,767 - db - INFO - Database initialization completed. +2024-10-29 13:13:49,774 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.005541 seconds, for an average of 0.00138525 seconds per hash. +2024-10-29 13:13:49,774 - utils - INFO - Checking models directory... +2024-10-29 13:13:49,775 - utils - INFO - Models directory exists: /Users/sidmohan/Desktop/codexify/backend/src/models +2024-10-29 13:13:49,777 - utils - INFO - File already exists: /Users/sidmohan/Desktop/codexify/backend/src/models/nomic-embed-text-v1.5.Q6_K.gguf +2024-10-29 13:13:49,777 - utils - INFO - Model downloads completed. +2024-10-29 13:13:49,782 - db - INFO - Finished initializing set of input hash/llm_model_name combinations that are either currently being processed or have already been processed. Set size: 4; Took 0.004517 seconds, for an average of 0.00112925 seconds per hash. +2024-10-29 13:13:50,183 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 + +**Patient:** Emily Johns...' +2024-10-29 13:13:50,184 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.3 seconds. +2024-10-29 13:13:50,185 - functions - INFO - That's an average of 261.66 ms per sentence and 3.822 sentences per second (and 6.6956 total characters per ms) using pooling method 'mean' +2024-10-29 13:13:50,190 - utils - INFO - Building Faiss index over embeddings for model nomic-embed-text-v1.5.Q6_K with pooling method mean... +2024-10-29 13:13:50,191 - worker - INFO - Received request to find most similar strings for query text: `**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being` using model: nomic-embed-text-v1.5.Q6_K +2024-10-29 13:13:50,192 - worker - INFO - Computing embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:13:50,192 - functions - INFO - Received request for embedding for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' from IP address 'None' +2024-10-29 13:13:50,416 - functions - INFO - Sentence 1 of 1 has 1 embeddings for text '**Date:** April 10, 2024 **Patient:** Emily Johnso...' +2024-10-29 13:13:50,416 - functions - INFO - Calculated 768-dimensional embeddings (relative to the underlying token embedding dimensions of 768) for 1 sentences in a total of 0.2 seconds. +2024-10-29 13:13:50,417 - functions - INFO - That's an average of 221.83 ms per sentence and 4.508 sentences per second (and 7.8665 total characters per ms) using pooling method 'mean' +2024-10-29 13:13:50,417 - functions - INFO - Embedding calculated for '**Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being' using model 'nomic-embed-text-v1.5.Q6_K' and embedding pooling method 'mean' in 0.00 seconds, or an average of 0.00 seconds per word. Now saving to database... +2024-10-29 13:13:50,418 - worker - INFO - Computed embedding for input text: **Date:** April 10, 2024 **Patient:** Emily Johnson, 35 years old **MRN:** 00987654 **Chief Complaint:** "I've been experiencing severe back pain and numbness in my legs" **History of Present Illness:** The patient is a 35-year-old who presents with a 2-month history of worsening back pain, numbness in both legs, and occasional tingling sensations The patient reports working as a freelance writer and has been experiencing increased stress due to tight deadlines and financial struggles **Past Medical History:** Hypothyroidism **Social History:** The patient shares a small apartment with two roommates and relies on public transportation They mention feeling overwhelmed with work and personal responsibilities, often sacrificing sleep to meet deadlines The patient expresses concern over the high cost of healthcare and the need for affordable medication options **Review of Systems:** Denies fever, chest pain, or shortness of breath Reports occasional headaches **Physical Examination:** - General: Appears tired but is alert and oriented - Vitals: BP 128/80, HR 72, Temp 98.6F, Resp 14/min **Assessment/Plan:** - Continue to monitor blood pressure and thyroid function - Discuss affordable medication options with a pharmacist - Refer to a social worker to address housing concerns and access to healthcare services - Encourage the patient to engage with community support groups for social support - Schedule a follow-up appointment in 4 weeks or sooner if symptoms worsen **Comments:** The patient's health concerns are compounded by socioeconomic factors, including employment status, housing stability, and access to healthcare Addressing these social determinants of health is crucial for improving the patient's overall well-being +2024-10-29 13:13:50,435 - worker - INFO - Finished advanced search in 0.249137 seconds. Found 1 results. +2024-10-29 13:13:50,440 - rq.worker - INFO - document_scans: Job OK (c802d85a865e070edc18f7147c003a1f) +2024-10-29 13:13:50,441 - rq.worker - INFO - Result is kept for 86400 seconds diff --git a/src/worker.py b/src/worker.py index 1669f59..8c88b93 100644 --- a/src/worker.py +++ b/src/worker.py @@ -4,7 +4,7 @@ from rq import Worker, Queue, Connection, get_current_job import logging import multiprocessing as mp -from functions import parse_submitted_document_file_into_sentence_strings_func +from functions import parse_submitted_document_file_into_sentence_strings_func, get_or_compute_embedding, get_list_of_corpus_identifiers_from_list_of_embedding_texts, prepare_string_for_embedding import traceback import platform import magika @@ -21,8 +21,15 @@ from datetime import datetime import json from models import Document, DocumentEmbedding -from db import AsyncSessionLocal +from db import AsyncSessionLocal, DatabaseWriter import urllib +from fastapi import HTTPException +from models import EmbeddingRequest +import numpy as np +import fast_vector_similarity as fvs +from utils import build_faiss_indexes +import faiss +from typing import Optional # Configure logging before anything else logging.basicConfig( @@ -266,6 +273,156 @@ async def upload_file_task(file_path_or_url: str, hash: str, size: int, llm_mode if os.path.exists(file_path_or_url) and file_path_or_url.startswith('/tmp/'): os.remove(file_path_or_url) +async def scan_document_task( + document_hash: Optional[str], + llm_model_name: str, + embedding_pooling_method: str, + corpus_identifier_string: str, + json_format: str, + send_back_json_or_zip_file: str, + query_text: str, + similarity_filter_percentage: float = 0.01, + number_of_most_similar_strings_to_return: int = 10, + result_sorting_metric: str = "hoeffding_d" +) -> dict: + job = get_current_job() + job.meta['progress'] = 0 + job.save_meta() + + logger.info(f"Starting document scan task for document hash: {document_hash}") + try: + # Initialize Redis manager + redis_manager = RedisManager() + await redis_manager.initialize() + + client_ip = "localhost" + + # Initialize database writer and start its processing loop + queue = asyncio.Queue() + db_writer = DatabaseWriter(queue) + await db_writer.initialize_processing_hashes() + + # Start the database writer task and wait for it to be ready + db_writer_task = asyncio.create_task(db_writer.dedicated_db_writer()) + await asyncio.sleep(0.1) # Give the writer task time to start + + # Create embedding request + embedding_request = EmbeddingRequest( + text=query_text, + llm_model_name=llm_model_name, + embedding_pooling_method=embedding_pooling_method + ) + + try: + # Get embedding without passing db_writer since it's not needed for this operation + embedding_response = await get_or_compute_embedding( + request=embedding_request, + use_verbose=False, + client_ip=client_ip + ) + + global faiss_indexes, associated_texts_by_model_and_pooling_method + request_time = datetime.utcnow() + query_text = prepare_string_for_embedding(query_text) + unique_id = f"advanced_semantic_search_{query_text}_{llm_model_name}_{embedding_pooling_method}_{similarity_filter_percentage}_{number_of_most_similar_strings_to_return}" + + faiss_indexes, associated_texts_by_model_and_pooling_method = await build_faiss_indexes(force_rebuild=True) + try: + faiss_index = faiss_indexes[(llm_model_name, embedding_pooling_method)] + except KeyError: + raise HTTPException(status_code=400, detail=f"No FAISS index found for model: {llm_model_name} and pooling method: {embedding_pooling_method}") + + num_results_before_corpus_filter = number_of_most_similar_strings_to_return * 25 + logger.info(f"Received request to find most similar strings for query text: `{query_text}` using model: {llm_model_name}") + try: + logger.info(f"Computing embedding for input text: {query_text}") + embedding_request = EmbeddingRequest(text=query_text, llm_model_name=llm_model_name, embedding_pooling_method=embedding_pooling_method) + embedding_response = await get_or_compute_embedding(embedding_request, db_writer=db_writer) # Pass db_writer here + embedding_json = embedding_response["text_embedding_dict"]["embedding_json"] + embedding_vector = json.loads(embedding_json) + input_embedding = np.array(embedding_vector).astype('float32').reshape(1, -1) + faiss.normalize_L2(input_embedding) + logger.info(f"Computed embedding for input text: {query_text}") + final_results = [] + + if faiss_index is None: + raise HTTPException(status_code=400, detail=f"No FAISS index found for model: {llm_model_name} and pooling method: {embedding_pooling_method}") + + num_results = max([1, int((1 - similarity_filter_percentage) * len(associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method]))]) + num_results_before_corpus_filter = min(num_results_before_corpus_filter, len(associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method])) + similarities, indices = faiss_index.search(input_embedding, num_results_before_corpus_filter) + filtered_indices = indices[0] + filtered_similarities = similarities[0] + similarity_results = [] + associated_texts = associated_texts_by_model_and_pooling_method[llm_model_name][embedding_pooling_method] + list_of_corpus_identifier_strings = await get_list_of_corpus_identifiers_from_list_of_embedding_texts(associated_texts, llm_model_name, embedding_pooling_method) + + for idx, similarity in zip(filtered_indices, filtered_similarities): + if idx < len(associated_texts) and list_of_corpus_identifier_strings[idx] == corpus_identifier_string: + associated_text = associated_texts[idx] + similarity_results.append((similarity, associated_text)) + + similarity_results = sorted(similarity_results, key=lambda x: x[0], reverse=True)[:num_results] + + for _, associated_text in similarity_results: + embedding_request = EmbeddingRequest(text=associated_text, llm_model_name=llm_model_name, embedding_pooling_method=embedding_pooling_method) + embedding_response = await get_or_compute_embedding(request=embedding_request, db_writer=db_writer, use_verbose=False) + embedding_json = embedding_response["text_embedding_dict"]["embedding_json"] + embedding_vector = json.loads(embedding_json) + comparison_embedding = np.array(embedding_vector).astype('float32').reshape(1, -1) + params = { + "vector_1": input_embedding.tolist()[0], + "vector_2": comparison_embedding.tolist()[0], + "similarity_measure": "all" + } + similarity_stats_str = fvs.py_compute_vector_similarity_stats(json.dumps(params)) + similarity_stats_json = json.loads(similarity_stats_str) + final_results.append({ + "search_result_text": associated_text, + "similarity_to_query_text": similarity_stats_json + }) + + num_to_return = number_of_most_similar_strings_to_return if number_of_most_similar_strings_to_return is not None else len(final_results) + results = sorted(final_results, key=lambda x: x["similarity_to_query_text"][result_sorting_metric], reverse=True)[:num_to_return] + + response_time = datetime.utcnow() + total_time = (response_time - request_time).total_seconds() + logger.info(f"Finished advanced search in {total_time} seconds. Found {len(results)} results.") + + return { + "query_text": query_text, + "corpus_identifier_string": corpus_identifier_string, + "embedding_pooling_method": embedding_pooling_method, + "results": results + } + + except Exception as e: + logger.error(f"An error occurred while processing the request: {e}") + traceback.print_exc() + raise HTTPException(status_code=500, detail="Internal Server Error") + + finally: + # Clean up the db_writer_task when done + if not db_writer_task.done(): + db_writer_task.cancel() + try: + await db_writer_task + except asyncio.CancelledError: + pass + + except Exception as e: + error_msg = f"Error in scan task: {str(e)}" + logger.error(error_msg) + logger.error(traceback.format_exc()) + job.meta['progress'] = 100 + job.save_meta() + return { + "status": "error", + "message": error_msg + } + finally: + job.meta['progress'] = 100 + job.save_meta() class MultiQueueWorker: def __init__(self): setup_process()