docker run -p 8181:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/weaviate:1.24.10
{"action":"startup","default_vectorizer_module":"none","level":"info","msg":"the default vectorizer modules is set to \"none\", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer","time":"2024-05-03T15:53:46Z"}
{"action":"startup","auto_schema_enabled":true,"level":"info","msg":"auto schema enabled setting is set to \"true\"","time":"2024-05-03T15:53:46Z"}
{"level":"info","msg":"No resource limits set, weaviate will use all available memory and CPU. To limit resources, set LIMIT_RESOURCES=true","time":"2024-05-03T15:53:46Z"}
{"action":"grpc_startup","level":"info","msg":"grpc server listening at [::]:50051","time":"2024-05-03T15:53:46Z"}
{"action":"restapi_management","level":"info","msg":"Serving weaviate at http://[::]:8080","time":"2024-05-03T15:53:46Z"}
{"action":"telemetry_push","level":"info","msg":"telemetry started","payload":"\u0026{MachineID:e7f2e3d0-cafe-40b9-b3b1-6df88f74a27a Type:INIT Version:1.24.10 Modules: NumObjects:0 OS:linux Arch:amd64}","time":"2024-05-03T15:53:47Z"}

#dspy-ai==2.4.9  
#weaviate-client==4.5.7

import dspy
llama3_ollama = dspy.OllamaLocal(model="llama3:8b-instruct-q6_K", max_tokens=4000, timeout_s=480)

import weaviate
from dspy.retrieve.weaviate_rm import WeaviateRM
weaviate_client = weaviate.connect_to_local(port=8181)
retriever_model = WeaviateRM("WeaviateBlogChunk", weaviate_client=weaviate_client, k=10)

dspy.settings.configure(lm=llama3_ollama, rm=retriever_model)
llama3_ollama("say hello")
["Hello! It's nice to meet you. Is there something I can help you with, or would you like to chat?"]

import json

file_path = './WeaviateBlogRAG-0-0-0.json'
with open(file_path, 'r') as file:
    dataset = json.load(file)

gold_answers = []
queries = []

for row in dataset:
    gold_answers.append(row["gold_answer"])
    queries.append(row["query"])
    
data = []

for i in range(len(gold_answers)):
    data.append(dspy.Example(gold_answer=gold_answers[i], question=queries[i]).with_inputs("question"))

trainset, devset, testset = data[:25], data[25:35], data[35:]

class TypedEvaluator(dspy.Signature):
    """Evaluate the quality of a system's answer to a question according to a given criterion."""
    
    criterion: str = dspy.InputField(desc="The evaluation criterion.")
    question: str = dspy.InputField(desc="The question asked to the system.")
    ground_truth_answer: str = dspy.InputField(desc="An expert written Ground Truth Answer to the question.")
    predicted_answer: str = dspy.InputField(desc="The system's answer to the question.")
    rating: float = dspy.OutputField(desc="A float rating between 1 and 5. IMPORTANT!! ONLY OUTPUT THE RATING!!")

def MetricWrapper(gold, pred, trace=None):
    alignment_criterion = "How aligned is the predicted_answer with the ground_truth?"
    return dspy.TypedPredictor(TypedEvaluator)(criterion=alignment_criterion,
                                          question=gold.question,
                                          ground_truth_answer=gold.gold_answer,
                                          predicted_answer=pred.answer).rating

class GenerateAnswer(dspy.Signature):
    """Assess the the context and answer the question."""

    context = dspy.InputField(desc="Helpful information for answering the question.")
    question = dspy.InputField()
    answer = dspy.OutputField(desc="A detailed answer that is supported by the context. ONLY OUTPUT THE ANSWER!!")
    
class RAG(dspy.Module):
    def __init__(self, k=3):
        super().__init__()
        
        self.retrieve = dspy.Retrieve(k=k)
        self.generate_answer = dspy.Predict(GenerateAnswer)
    
    def forward(self, question):
        context = self.retrieve(question).passages
        pred = self.generate_answer(context=context, question=question).answer
        return dspy.Prediction(context=context, answer=pred, question=question)

print(RAG()("What is binary quantization?").answer)

---------------------------------------------------------------------------
_InactiveRpcError                         Traceback (most recent call last)
File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/weaviate/collections/grpc/query.py:609, in _QueryGRPC.__call(self, request)
    608 res: search_get_pb2.SearchReply  # According to PEP-0526
--> 609 res, _ = self._connection.grpc_stub.Search.with_call(
    610     request,
    611     metadata=self._connection.grpc_headers(),
    612     timeout=self._connection.timeout_config.query,
    613 )
    615 return res

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/grpc/_channel.py:1198, in _UnaryUnaryMultiCallable.with_call(self, request, timeout, metadata, credentials, wait_for_ready, compression)
   1192 (
   1193     state,
   1194     call,
   1195 ) = self._blocking(
   1196     request, timeout, metadata, credentials, wait_for_ready, compression
   1197 )
-> 1198 return _end_unary_response_blocking(state, call, True, None)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/grpc/_channel.py:1006, in _end_unary_response_blocking(state, call, with_call, deadline)
   1005 else:
-> 1006     raise _InactiveRpcError(state)

_InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
	status = StatusCode.UNKNOWN
	details = "extract params: no such class with name 'WeaviateBlogChunk' found in the schema. Check your schema files for which classes are available"
	debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"extract params: no such class with name \'WeaviateBlogChunk\' found in the schema. Check your schema files for which classes are available", grpc_status:2, created_time:"2024-05-03T18:50:50.803414318+03:00"}"
>

During handling of the above exception, another exception occurred:

WeaviateQueryError                        Traceback (most recent call last)
Cell In[7], line 1
----> 1 print(RAG()("What is binary quantization?").answer)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dspy/primitives/program.py:26, in Module.__call__(self, *args, **kwargs)
     25 def __call__(self, *args, **kwargs):
---> 26     return self.forward(*args, **kwargs)

Cell In[6], line 16, in RAG.forward(self, question)
     15 def forward(self, question):
---> 16     context = self.retrieve(question).passages
     17     pred = self.generate_answer(context=context, question=question).answer
     18     return dspy.Prediction(context=context, answer=pred, question=question)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dspy/retrieve/retrieve.py:30, in Retrieve.__call__(self, *args, **kwargs)
     29 def __call__(self, *args, **kwargs):
---> 30     return self.forward(*args, **kwargs)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dspy/retrieve/retrieve.py:39, in Retrieve.forward(self, query_or_queries, k, **kwargs)
     36 # print(queries)
     37 # TODO: Consider removing any quote-like markers that surround the query too.
     38 k = k if k is not None else self.k
---> 39 passages = dsp.retrieveEnsemble(queries, k=k,**kwargs)
     40 return Prediction(passages=passages)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dsp/primitives/search.py:57, in retrieveEnsemble(queries, k, by_prob, **kwargs)
     54 queries = [q for q in queries if q]
     56 if len(queries) == 1:
---> 57     return retrieve(queries[0], k, **kwargs)
     59 passages = {}
     60 for q in queries:

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dsp/primitives/search.py:12, in retrieve(query, k, **kwargs)
     10 if not dsp.settings.rm:
     11     raise AssertionError("No RM is loaded.")
---> 12 passages = dsp.settings.rm(query, k=k, **kwargs)
     13 if not isinstance(passages, Iterable):
     14     # it's not an iterable yet; make it one.
     15     # TODO: we should unify the type signatures of dspy.Retriever
     16     passages = [passages]

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dspy/retrieve/retrieve.py:30, in Retrieve.__call__(self, *args, **kwargs)
     29 def __call__(self, *args, **kwargs):
---> 30     return self.forward(*args, **kwargs)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/dspy/retrieve/weaviate_rm.py:81, in WeaviateRM.forward(self, query_or_queries, k)
     79 for query in queries:
     80     if self._client_type == "WeaviateClient":
---> 81         results = self._weaviate_client.collections.get(self._weaviate_collection_name).query.hybrid(
     82             query=query,
     83             limit=k,
     84         )
     86         parsed_results = [result.properties[self._weaviate_collection_text_key] for result in results.objects]
     88     elif self._client_type == "Client":

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/weaviate/collections/queries/hybrid/query.py:84, in _HybridQuery.hybrid(self, query, alpha, vector, query_properties, fusion_type, limit, offset, auto_limit, filters, rerank, target_vector, include_vector, return_metadata, return_properties, return_references)
     19 def hybrid(
     20     self,
     21     query: Optional[str],
   (...)
     36     return_references: Optional[ReturnReferences[TReferences]] = None,
     37 ) -> QueryReturnType[Properties, References, TProperties, TReferences]:
     38     """Search for objects in this collection using the hybrid algorithm blending keyword-based BM25 and vector-based similarity.
     39 
     40     See the [docs](https://weaviate.io/developers/weaviate/search/hybrid) for a more detailed explanation.
   (...)
     82             If the network connection to Weaviate fails.
     83     """
---> 84     res = self._query.hybrid(
     85         query=query,
     86         alpha=alpha,
     87         vector=vector,
     88         properties=query_properties,
     89         fusion_type=fusion_type,
     90         limit=limit,
     91         offset=offset,
     92         autocut=auto_limit,
     93         filters=filters,
     94         rerank=rerank,
     95         target_vector=target_vector,
     96         return_metadata=self._parse_return_metadata(return_metadata, include_vector),
     97         return_properties=self._parse_return_properties(return_properties),
     98         return_references=self._parse_return_references(return_references),
     99     )
    100     return self._result_to_query_return(
    101         res,
    102         _QueryOptions.from_input(
   (...)
    111         return_references,
    112     )

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/weaviate/collections/grpc/query.py:207, in _QueryGRPC.hybrid(self, query, alpha, vector, properties, fusion_type, limit, offset, autocut, filters, return_metadata, return_properties, return_references, generative, rerank, target_vector)
    172 hybrid_search = (
    173     search_get_pb2.Hybrid(
    174         properties=properties,
   (...)
    191     else None
    192 )
    194 request = self.__create_request(
    195     limit=limit,
    196     offset=offset,
   (...)
    204     hybrid_search=hybrid_search,
    205 )
--> 207 return self.__call(request)

File ~/projects/sandbox/dspy-ollama/.venv/lib/python3.10/site-packages/weaviate/collections/grpc/query.py:618, in _QueryGRPC.__call(self, request)
    615     return res
    617 except grpc.RpcError as e:
--> 618     raise WeaviateQueryError(e.details(), "GRPC search")

WeaviateQueryError: Query call with protocol GRPC search failed with message extract params: no such class with name 'WeaviateBlogChunk' found in the schema. Check your schema files for which classes are available.

from dspy.teleprompt import MIPRO

import openai
gpt4 = dspy.OpenAI(model="gpt-4o", max_tokens=4000, model_type="chat")

teleprompter = MIPRO(prompt_model=gpt4, 
                     task_model=llama3_ollama, 
                     metric=MetricWrapper, 
                     num_candidates=3, 
                     init_temperature=0.5)
kwargs = dict(num_threads=1, 
              display_progress=True, 
              display_table=0)
MIPRO_compiled_RAG = teleprompter.compile(RAG(), trainset=trainset[:5], num_trials=3, max_bootstrapped_demos=1, max_labeled_demos=0, eval_kwargs=kwargs)

AttributeError: module 'wandb.proto.wandb_internal_pb2' has no attribute 'Result'

test_example = dspy.Example(question="What do cross encoders do?")
test_pred = dspy.Example(answer="They re-rank documents.")

type(llm_metric(test_example, test_pred))

test_example = dspy.Example(question="What do cross encoders do?")
test_pred = dspy.Example(answer="They re-rank documents.")

type(llm_metric(test_example, test_pred))