indexing.py

import os

from whoosh.fields import *
from whoosh.index import create_in
from whoosh import index

# Here, the structure of index entries is defined. You can add more fields with metadata, computed values etc.,
# and use them for searching and ranking.
# We only use a title and a text.
#
# The "stored" attribute is used for all parts that we want to be able to fully retrieve from the index

ix_schema = Schema(title=TEXT(stored=True), url=ID(stored=True, unique=True), content=TEXT(stored=True), date=DATETIME(stored=True))

# Create an index if not created or open an existing, then return it
def get_index():
    if not os.path.exists("indexdir"):
        os.makedirs("indexdir")
        return create_in("indexdir", schema=ix_schema)
    return index.open_dir("indexdir")