Add IDF to scoring (vary basic).

Author: Alexander Korotkov

src/rum_ts_utils.c

@@ -101,18 +101,25 @@ typedef struct
 
 typedef struct
 {
-	bool	operandexist;
+	bool			operandexist;
 	WordEntryPos	pos;
 }
 QueryRepresentationOperand;
 
+typedef struct
+{
+	float4			idf;
+	bool			idfloaded;
+} QueryRepresentationIDF;
+
 typedef struct
 {
 	TSQuery		query;
 	/* Used in rum_tsquery_distance() */
 	int		   *map_item_operand;
 
 	QueryRepresentationOperand	   *operandData;
+	QueryRepresentationIDF		   *operandIdf;
 	int			length;
 } QueryRepresentation;
 
@@ -140,6 +147,7 @@ static WordEntryPosVector POSNULL = {
 #define RANK_NORM_UNIQ			0x08
 #define RANK_NORM_LOGUNIQ		0x10
 #define RANK_NORM_RDIVRPLUS1	0x20
+#define RANK_NORM_IDF			0x40
 #define DEF_NORM_METHOD			RANK_NO_NORM
 
 #define QR_GET_OPERAND(q, v)	\
@@ -1229,6 +1237,7 @@ calc_score_docr(float4 *arrdata, DocRepresentation *doc, uint32 doclen,
 	{
 		double		Cpos = 0.0;
 		double		InvSum = 0.0;
+		double		Idf = 0.0;
 		int			nNoise;
 		DocRepresentation *ptr = ext.begin;
 		/* Added by SK */
@@ -1278,13 +1287,43 @@ calc_score_docr(float4 *arrdata, DocRepresentation *doc, uint32 doclen,
 
 		/* Compute the number of query terms in the cover */
 		for (i = 0; i < qr->length; i++)
+		{
 			if (qr->operandData[i].operandexist)
-				nitems++;
+			{
+				if (method & RANK_NORM_IDF)
+				{
+					if (!qr->operandIdf[i].idfloaded)
+					{
+						QueryOperand *oper = (QueryOperand *) (GETQUERY(qr->query) + i);
+						qr->operandIdf[i].idf =
+							estimate_idf(
+								GETOPERAND(qr->query) + oper->distance,
+								oper->length
+							);
+						qr->operandIdf[i].idfloaded = true;
+					}
+
+					Idf += qr->operandIdf[i].idf;
+				}
+				else
+				{
+					nitems++;
+				}
+			}
+		}
 
 		Cpos = ((double) (ext.end - ext.begin + 1)) / InvSum;
 
-		if (nitems > 0)
-			Cpos *= nitems;
+		if (method & RANK_NORM_IDF)
+		{
+			if (Idf >= 1.0)
+				Cpos *= Idf;
+		}
+		else
+		{
+			if (nitems > 0)
+				Cpos *= nitems;
+		}
 
 		/*
 		 * if doc are big enough then ext.q may be equal to ext.p due to limit
@@ -1369,6 +1408,8 @@ calc_score(float4 *arrdata, TSVector txt, TSQuery query, int method)
 	qr.query = query;
 	qr.map_item_operand = NULL;
 	qr.operandData = palloc0(sizeof(qr.operandData[0]) * query->size);
+	if (method & RANK_NORM_IDF)
+		qr.operandIdf = palloc0(sizeof(qr.operandIdf[0]) * query->size);
 	qr.length = query->size;
 
 	doc = get_docrep(txt, &qr, &doclen);

src/tf_idf.c

@@ -12,6 +12,7 @@
 #include "catalog/namespace.h"
 #include "catalog/pg_statistic.h"
 #include "catalog/pg_type.h"
+#include "nodes/nodeFuncs.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
@@ -20,6 +21,12 @@
 
 #include "rum.h"
 
+/*
+ * FIXME:
+ *  * cache IDF
+ *  * handle prefix search
+ */
+
 /* lookup table type for binary searching through MCELEMs */
 typedef struct
 {
@@ -77,7 +84,6 @@ check_tf_idf_source(char **newval, void **extra, GucSource source)
 	Oid					namespaceId;
 	Oid					relId;
 	Relation			rel = NULL;
-	TupleDesc			tupDesc;
 	AttrNumber			attrno;
 	int					i;
 	RelAttrInfo		   *myextra;
@@ -119,17 +125,27 @@ check_tf_idf_source(char **newval, void **extra, GucSource source)
 		EXIT_CHECK_TF_IDF_SOURCE("relation not found");
 
 	rel = RelationIdGetRelation(relId);
-	tupDesc = rel->rd_att;
 	if (rel->rd_rel->relkind == RELKIND_INDEX)
 	{
+		int		exprnum = 0;
+
 		attrno = pg_atoi(attname, sizeof(attrno), 10);
 		if (attrno <= 0 || attrno > rel->rd_index->indnatts)
 			EXIT_CHECK_TF_IDF_SOURCE("wrong index attribute number");
 		if (rel->rd_index->indkey.values[attrno - 1] != InvalidAttrNumber)
 			EXIT_CHECK_TF_IDF_SOURCE("regular indexed column is specified");
+		for (i = 0; i < attrno - 1; i++)
+		{
+			if (rel->rd_index->indkey.values[i] == InvalidAttrNumber)
+				exprnum++;
+		}
+		if (exprType((Node *) list_nth(rel->rd_indexprs, exprnum)) != TSVECTOROID)
+			EXIT_CHECK_TF_IDF_SOURCE("indexed expression should be of tsvector type");
 	}
 	else
 	{
+		TupleDesc	tupDesc = rel->rd_att;
+
 		attrno = InvalidAttrNumber;
 		for (i = 0; i < tupDesc->natts; i++)
 		{
@@ -139,13 +155,12 @@ check_tf_idf_source(char **newval, void **extra, GucSource source)
 				break;
 			}
 		}
-
 		if (attrno == InvalidAttrNumber)
 			EXIT_CHECK_TF_IDF_SOURCE("attribute not found");
+		if (tupDesc->attrs[attrno - 1]->atttypid != TSVECTOROID)
+			EXIT_CHECK_TF_IDF_SOURCE("attribute should be of tsvector type");
 	}
 
-	if (tupDesc->attrs[attrno - 1]->atttypid != TSVECTOROID)
-		EXIT_CHECK_TF_IDF_SOURCE("attribute should be of tsvector type");
 
 	myextra = (RelAttrInfo *) malloc(sizeof(RelAttrInfo));
 	myextra->relId = relId;
@@ -164,7 +179,16 @@ assign_tf_idf_source(const char *newval, void *extra)
 {
 	RelAttrInfo  *myextra = (RelAttrInfo *) extra;
 
-	TFIDFSourceParsed = *myextra;
+	if (myextra)
+	{
+		TFIDFSourceParsed = *myextra;
+	}
+	else
+	{
+		TFIDFSourceParsed.relId = InvalidOid;
+		TFIDFSourceParsed.attrno = InvalidAttrNumber;
+	}
+
 	forget_tf_idf_stats();
 }
 
@@ -181,6 +205,15 @@ load_tf_idf_source(void)
 											 "Memory context for TF/IDF statistics",
 											 ALLOCSET_DEFAULT_SIZES);
 
+	if (!OidIsValid(TFIDFSourceParsed.relId)
+		|| TFIDFSourceParsed.attrno == InvalidAttrNumber)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("statistics for TD/IDF is not defined"),
+				 errhint("consider setting tf_idf_source GUC")));
+	}
+
 	statsTuple = SearchSysCache3(STATRELATTINH,
 								 ObjectIdGetDatum(TFIDFSourceParsed.relId),
 								 Int16GetDatum(TFIDFSourceParsed.attrno),
@@ -228,6 +261,8 @@ load_tf_idf_source(void)
 
 	MemoryContextSwitchTo(oldContext);
 
+	TDIDFLoaded = true;
+
 	ReleaseSysCache(statsTuple);
 }
 
@@ -241,7 +276,8 @@ check_load_tf_idf_source(void)
 static void
 forget_tf_idf_stats(void)
 {
-	MemoryContextReset(TFIDFContext);
+	if (TFIDFContext)
+		MemoryContextReset(TFIDFContext);
 	TDIDFLoaded = false;
 }