Merge branch 'master' of https://gitlab.fit.cvut.cz/koristo1/vwm-vector-model

src/preprocessing/database/create-script.sql

 CREATE TABLE Document
 (
     id       INTEGER PRIMARY KEY AUTOINCREMENT,
-    filename TEXT NOT NULL
+    filename TEXT NOT NULL UNIQUE
 );
 CREATE TABLE Term
 (
     id    INTEGER PRIMARY KEY AUTOINCREMENT,
-    value TEXT NOT NULL
+    value TEXT NOT NULL UNIQUE
 );
 CREATE TABLE TermDocumentOccurrence
@@ -19,5 +19,3 @@ CREATE TABLE TermDocumentOccurrence
     FOREIGN KEY (Document_id) REFERENCES Document (id),
     FOREIGN KEY (Term_id) REFERENCES Term (id)
 );
-
-

src/preprocessing/preprocessor.py

@@ -137,7 +137,10 @@ class Preprocessor:
         document_key = database.last_primary_key()
         for term in self.terms:
             database.execute('''INSERT OR IGNORE INTO Term(value) VALUES (?)''', [term])
-            term_key = database.last_primary_key()
+            database.execute('''SELECT id FROM Term WHERE value = ?''', [term])
+            term_key = database.fetchone()[0]
+            if term_key is None:
+                term_key = database.last_primary_key()
             database.execute('''INSERT INTO TermDocumentOccurrence(Term_id, Document_id, count) VALUES (?,?,?)''',
                              [term_key, document_key, self.terms[term]])
         database.commit()