Image retrieval using Lindorm's multimodal capabilities

Retrieval method

Description

Engines involved

Search by text

Search for images by using text. You can use pure vector search or full-text and vector hybrid search.

1. Enter a text description.

   def on_button_clicked(b):
       with output:
           clear_output()  # Clear the previous output.
           if text_input.value:
               code, embedding = lindorm.text_embedding(text_input.value)
           else:
               print("Please enter a description first.")
               return
           
           if b.description == "Pure Vector Search":
               hits = lindorm.knn_search(embedding)
           elif b.description == "RRF Hybrid Search":
               if text_input.value == '':
                   print("Please enter keywords first.")
                   return
               hits = lindorm.rrf_search(text_input.value, embedding)
           print("Search by text")
           show_hits(hits)

2. Convert text to a vector.

   if text_input.value:
               code, embedding = lindorm.text_embedding(text_input.value)

   def text_embedding(self, text: str):
           data = {"input": [text]}
           return self.post_model_request(Config.LD_TEXT_MODEL, data)
        

3. Retrieve images.

   • Vector search.

     if b.description == "Pure Vector Search":
         hits = lindorm.knn_search(embedding)

     def knn_search(self, picture_embedding):
             body = {
                 "query": {
                     "knn": {
                         "image_embedding": {
                             "vector": picture_embedding,
                             "k": self.top_k
                         }
                     }
                 },
                 "size": self.top_k
             }
             response = self.client.search(index=self.index_name, body=body)
             return response.get('hits').get('hits')
     

   • Hybrid search (full-text and vector).

     elif b.description == "RRF Hybrid Search":
         if text_input.value == '':
             print("Please enter keywords first.")
             return
         hits = lindorm.rrf_search(text_input.value, embedding)

     def rrf_search(self, text, embedding):
             query = {
                 "_source": ["description"],
                 "size": self.top_k,
                 "query": {
                     "knn": {
                         "image_embedding": {
                             "vector": embedding,
                             "filter": {
                                 "match": {
                                     "description": text
                                 }
                             },
                             "k": self.top_k
                         }
                     }
                 },
                 "ext": {"lvector": {
                     "hybrid_search_type": "filter_rrf",
                     "rrf_rank_constant": "60",
                     "rrf_knn_weight_factor": "0.4"
                 }}
             }
             response = self.client.search(index=self.index_name, body=query)
             return response.get('hits').get('hits')
             

AI Engine

Search Engine

Vector Engine

Search by image

Search for images by using an image. This method performs a vector search.

1. Display the uploaded image.

   def show_uploaded_image(change):
       with output:
           clear_output()  # Clear the previous output.
           if change['new']:
               # Get the content of the uploaded file.
               uploaded_file = change['new'][0]
               content = uploaded_file['content']
               # Open the image by using PIL.
               image = Image.open(io.BytesIO(content))
               # Display the image by using matplotlib.
               plt.imshow(image)
               plt.axis('off')  # Do not display the axes.
               plt.show()
               code, embedding = lindorm.picture_embedding(content)
               hits = lindorm.knn_search(embedding)
               print("Image Upload Search")
               show_hits(hits)

2. Convert the image to a vector.

   code , embedding = lindorm.picture_embedding(content)

   def picture_embedding(self, image):
       encoded_str = base64.b64encode(image).decode('utf-8')
       data = {"input": {"images": [encoded_str]}}
       return self.post_model_request(Config.LD_VL_MODEL, data)

3. Retrieve images by performing a vector search.

   hits = lindorm.knn_search(embedding)

   def knn_search(self, picture_embedding):
           body = {
               "query": {
                   "knn": {
                       "image_embedding": {
                           "vector": picture_embedding,
                           "k": self.top_k
                       }
                   }
               },
               "size": self.top_k
           }
           response = self.client.search(index=self.index_name, body=body)
           return response.get('hits').get('hits')
       

AI Engine

Search Engine

Vector Engine

Search by image - image deduplication

Run LindormMultiModalSearch.ipynb to perform image deduplication.

1. Create an index and ingest images.

   if __name__ == '__main__':
       print("start")
       lindorm = Lindorm("multimodal_search_index2")
       
       if lindorm.get_index() is None:
           # lindorm.drop_index()
           lindorm.create_search_index()
       print("Index created.")
       import_all_data(dir_input.value)

2. Calculate image similarity.

   # Calculate the image vector.
   code, picture_embedding = lindorm.picture_embedding(image_data)
   hits = lindorm.knn_search(picture_embedding)
   
   # Display float_input.
   print('Image similarity threshold:', float_input.value)
   print('Image type:', image_type)

3. If a similar image that meets the threshold already exists, the new image is not imported. Otherwise, it is added to the database.

   if hits[0].get('_score') >= float_input.value:
       print(f"An image with a similarity score >= {float_input.value} already exists. The new image will not be imported.")
   else:
       print(f"No image with a similarity score >= {float_input.value} exists. Importing this image.")
       dir = dir_input.value
       if not os.path.exists(dir):
           os.makedirs(dir)
       pic_name = str(uuid.uuid4()) + "." + image_type
       pic_path = os.path.join(dir, f"{os.path.basename(pic_name)}")
       import_new_image(image_data, pic_path, picture_embedding)

AI Engine

Search Engine

Vector Engine