diff --git a/app/controllers/EdgeController.java b/app/controllers/EdgeController.java index 060910f9..85e97c13 100644 --- a/app/controllers/EdgeController.java +++ b/app/controllers/EdgeController.java @@ -5,11 +5,6 @@ import dao.models.EdgeVersionFactory; import db.DbClient; import exceptions.GroundException; -import exceptions.GroundItemNotFoundException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.inject.Inject; import models.models.Edge; import models.models.EdgeVersion; import models.models.Tag; @@ -19,6 +14,10 @@ import util.ControllerUtils; import util.FactoryGenerator; +import javax.inject.Inject; +import java.util.List; +import java.util.Map; + public class EdgeController extends Controller { private final EdgeFactory edgeFactory; private final EdgeVersionFactory edgeVersionFactory; diff --git a/app/controllers/GraphController.java b/app/controllers/GraphController.java index 84b31b8e..b724d588 100644 --- a/app/controllers/GraphController.java +++ b/app/controllers/GraphController.java @@ -6,10 +6,6 @@ import db.DbClient; import exceptions.GroundException; import exceptions.GroundItemNotFoundException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.inject.Inject; import models.models.Graph; import models.models.GraphVersion; import models.models.Tag; @@ -19,6 +15,11 @@ import util.ControllerUtils; import util.FactoryGenerator; +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class GraphController extends Controller { private final GraphFactory graphFactory; private final GraphVersionFactory graphVersionFactory; diff --git a/app/controllers/NodeController.java b/app/controllers/NodeController.java index 2161ec93..93e22b50 100644 --- a/app/controllers/NodeController.java +++ b/app/controllers/NodeController.java @@ -7,20 +7,20 @@ import db.DbClient; import exceptions.GroundException; import exceptions.GroundItemNotFoundException; -import java.util.List; -import java.util.Map; -import javax.inject.Inject; import models.models.Node; import models.models.NodeVersion; import models.models.Tag; import play.libs.Json; import play.mvc.Controller; import play.mvc.Result; - -import java.util.HashMap; import util.ControllerUtils; import util.FactoryGenerator; +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class NodeController extends Controller { private final NodeFactory nodeFactory; private final NodeVersionFactory nodeVersionFactory; @@ -106,4 +106,4 @@ public Result createNodeVersion(String sourceKey) throws GroundException { throw e; } } -} \ No newline at end of file +} diff --git a/app/dao/models/cassandra/CassandraRichVersionFactory.java b/app/dao/models/cassandra/CassandraRichVersionFactory.java index dbcdff13..6a0c01d4 100644 --- a/app/dao/models/cassandra/CassandraRichVersionFactory.java +++ b/app/dao/models/cassandra/CassandraRichVersionFactory.java @@ -26,6 +26,7 @@ import models.models.StructureVersion; import models.models.Tag; import models.versions.GroundType; +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; @@ -93,7 +94,7 @@ public void insertIntoDatabase(long id, for (String key : tags.keySet()) { Tag tag = tags.get(key); - + ElasticSearch.insertElasticSearch(tag, "rich_version"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id)); tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key)); diff --git a/app/dao/models/cassandra/CassandraTagFactory.java b/app/dao/models/cassandra/CassandraTagFactory.java index fa0dde4a..4f50e2d9 100644 --- a/app/dao/models/cassandra/CassandraTagFactory.java +++ b/app/dao/models/cassandra/CassandraTagFactory.java @@ -12,6 +12,7 @@ * limitations under the License. */ + package dao.models.cassandra; import dao.models.TagFactory; @@ -22,7 +23,7 @@ import exceptions.GroundException; import models.models.Tag; import models.versions.GroundType; - +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -30,9 +31,11 @@ public class CassandraTagFactory implements TagFactory { private final CassandraClient dbClient; + private final boolean elasticSearchOn; - public CassandraTagFactory(CassandraClient dbClient) { + public CassandraTagFactory(CassandraClient dbClient, boolean elasticSearchOn) { this.dbClient = dbClient; + this.elasticSearchOn = elasticSearchOn; } @Override @@ -75,16 +78,24 @@ private Map retrieveFromDatabaseById(long id, String keyPrefix) return result; } - @Override public List getVersionIdsByTag(String tag) throws GroundException { - return this.getIdsByTag(tag, "rich_version"); + if (elasticSearchOn) { + return ElasticSearch.getSearchResponse("rich_version", tag); + } else { + return this.getIdsByTag(tag, "rich_version"); + } } - @Override public List getItemIdsByTag(String tag) throws GroundException { - return this.getIdsByTag(tag, "item"); + if (elasticSearchOn) { + return ElasticSearch.getSearchResponse("item", tag); + } else { + return this.getIdsByTag(tag, "item"); + } } + + private List getIdsByTag(String tag, String keyPrefix) throws GroundException { List result = new ArrayList<>(); diff --git a/app/dao/models/neo4j/Neo4jRichVersionFactory.java b/app/dao/models/neo4j/Neo4jRichVersionFactory.java index bafe9226..3a8a6af2 100644 --- a/app/dao/models/neo4j/Neo4jRichVersionFactory.java +++ b/app/dao/models/neo4j/Neo4jRichVersionFactory.java @@ -12,29 +12,29 @@ * limitations under the License. */ + package dao.models.neo4j; import dao.models.RichVersionFactory; import dao.versions.neo4j.Neo4jVersionFactory; import db.DbDataContainer; import db.Neo4jClient; -import exceptions.GroundDbException; import exceptions.GroundException; import exceptions.GroundVersionNotFoundException; import models.models.RichVersion; import models.models.StructureVersion; import models.models.Tag; import models.versions.GroundType; +import org.neo4j.driver.internal.value.NullValue; +import org.neo4j.driver.internal.value.StringValue; +import org.neo4j.driver.v1.Record; +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.neo4j.driver.internal.value.NullValue; -import org.neo4j.driver.internal.value.StringValue; -import org.neo4j.driver.v1.Record; - public abstract class Neo4jRichVersionFactory extends Neo4jVersionFactory implements RichVersionFactory { @@ -104,7 +104,7 @@ public void insertIntoDatabase(long id, for (String key : tags.keySet()) { Tag tag = tags.get(key); - + ElasticSearch.insertElasticSearch(tag, "rich_version"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id)); tagInsertion.add(new DbDataContainer("tkey", GroundType.STRING, key)); diff --git a/app/dao/models/neo4j/Neo4jTagFactory.java b/app/dao/models/neo4j/Neo4jTagFactory.java index 5a736ef9..da5ae1ac 100644 --- a/app/dao/models/neo4j/Neo4jTagFactory.java +++ b/app/dao/models/neo4j/Neo4jTagFactory.java @@ -21,21 +21,23 @@ import exceptions.GroundException; import models.models.Tag; import models.versions.GroundType; +import org.neo4j.driver.internal.value.NullValue; +import org.neo4j.driver.internal.value.StringValue; +import org.neo4j.driver.v1.Record; +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.neo4j.driver.internal.value.NullValue; -import org.neo4j.driver.internal.value.StringValue; -import org.neo4j.driver.v1.Record; - public class Neo4jTagFactory implements TagFactory { private final Neo4jClient dbClient; + private final boolean elasticSearchOn; - public Neo4jTagFactory(Neo4jClient dbClient) { + public Neo4jTagFactory(Neo4jClient dbClient, boolean elasticSearchOn) { this.dbClient = dbClient; + this.elasticSearchOn = elasticSearchOn; } @Override @@ -86,14 +88,20 @@ private Map retrieveFromDatabaseById(long id, String keyPrefix) return tags; } - @Override - public List getVersionIdsByTag(String tag) throws GroundDbException { - return this.getIdsByTag(tag, "rich_version_id"); + public List getVersionIdsByTag(String tag) throws GroundException { + if (this.elasticSearchOn) { + return ElasticSearch.getSearchResponse("rich_version", tag); + } else { + return this.getIdsByTag(tag, "rich_version"); + } } - @Override - public List getItemIdsByTag(String tag) throws GroundDbException { - return this.getIdsByTag(tag, "item_id"); + public List getItemIdsByTag(String tag) throws GroundException { + if (this.elasticSearchOn) { + return ElasticSearch.getSearchResponse("item", tag); + } else { + return this.getIdsByTag(tag, "item"); + } } private List getIdsByTag(String tag, String idAttribute) throws GroundDbException { diff --git a/app/dao/models/postgres/PostgresRichVersionFactory.java b/app/dao/models/postgres/PostgresRichVersionFactory.java index 57916e26..b65fe1a6 100644 --- a/app/dao/models/postgres/PostgresRichVersionFactory.java +++ b/app/dao/models/postgres/PostgresRichVersionFactory.java @@ -12,6 +12,8 @@ * limitations under the License. */ + + package dao.models.postgres; import dao.models.RichVersionFactory; @@ -26,6 +28,7 @@ import models.models.StructureVersion; import models.models.Tag; import models.versions.GroundType; +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; @@ -92,7 +95,7 @@ public void insertIntoDatabase(long id, for (String key : tags.keySet()) { Tag tag = tags.get(key); - + ElasticSearch.insertElasticSearch(tag, "rich_version"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id)); tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key)); diff --git a/app/dao/models/postgres/PostgresTagFactory.java b/app/dao/models/postgres/PostgresTagFactory.java index 9fce0039..4615612e 100644 --- a/app/dao/models/postgres/PostgresTagFactory.java +++ b/app/dao/models/postgres/PostgresTagFactory.java @@ -12,6 +12,7 @@ * limitations under the License. */ + package dao.models.postgres; import dao.models.TagFactory; @@ -22,6 +23,7 @@ import exceptions.GroundException; import models.models.Tag; import models.versions.GroundType; +import util.ElasticSearch; import java.util.ArrayList; import java.util.HashMap; @@ -30,9 +32,11 @@ public class PostgresTagFactory implements TagFactory { private final PostgresClient dbClient; + private final boolean elasticSearchOn; - public PostgresTagFactory(PostgresClient dbClient) { + public PostgresTagFactory(PostgresClient dbClient, boolean elasticSearchOn) { this.dbClient = dbClient; + this.elasticSearchOn = elasticSearchOn; } @Override @@ -76,13 +80,21 @@ private Map retrieveFromDatabaseById(long id, String keyPrefix) @Override public List getVersionIdsByTag(String tag) throws GroundException { - return this.getIdsByTag(tag, "rich_version"); + if (this.elasticSearchOn) { + return ElasticSearch.getSearchResponse("rich_version", tag); + } else { + return this.getIdsByTag(tag, "rich_version"); + } } @Override public List getItemIdsByTag(String tag) throws GroundException { - return this.getIdsByTag(tag, "item"); + if (this.elasticSearchOn) { + return ElasticSearch.getSearchResponse("item", tag); + } else { + return this.getIdsByTag(tag, "item"); + } } private List getIdsByTag(String tag, String keyPrefix) throws GroundException { diff --git a/app/dao/versions/cassandra/CassandraItemFactory.java b/app/dao/versions/cassandra/CassandraItemFactory.java index eed950d7..c45d664e 100644 --- a/app/dao/versions/cassandra/CassandraItemFactory.java +++ b/app/dao/versions/cassandra/CassandraItemFactory.java @@ -25,14 +25,14 @@ import models.versions.GroundType; import models.versions.Item; import models.versions.VersionHistoryDag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import util.ElasticSearch; import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public abstract class CassandraItemFactory implements ItemFactory { private static final Logger LOGGER = LoggerFactory.getLogger(CassandraItemFactory.class); @@ -70,7 +70,7 @@ public void insertIntoDatabase(long id, Map tags) throws GroundExce for (String key : tags.keySet()) { Tag tag = tags.get(key); - + ElasticSearch.insertElasticSearch(tag, "item"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id)); tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key)); diff --git a/app/dao/versions/cassandra/CassandraVersionFactory.java b/app/dao/versions/cassandra/CassandraVersionFactory.java index 484cf204..47672c65 100644 --- a/app/dao/versions/cassandra/CassandraVersionFactory.java +++ b/app/dao/versions/cassandra/CassandraVersionFactory.java @@ -43,7 +43,6 @@ public CassandraVersionFactory(CassandraClient dbClient) { public void insertIntoDatabase(long id) throws GroundException { List insertions = new ArrayList<>(); insertions.add(new DbDataContainer("id", GroundType.LONG, id)); - this.dbClient.insert("version", insertions); } diff --git a/app/dao/versions/neo4j/Neo4jItemFactory.java b/app/dao/versions/neo4j/Neo4jItemFactory.java index 72660153..528f5213 100644 --- a/app/dao/versions/neo4j/Neo4jItemFactory.java +++ b/app/dao/versions/neo4j/Neo4jItemFactory.java @@ -14,8 +14,6 @@ package dao.versions.neo4j; -import org.neo4j.driver.v1.Record; - import dao.models.neo4j.Neo4jTagFactory; import dao.versions.ItemFactory; import db.DbDataContainer; @@ -27,6 +25,8 @@ import models.versions.GroundType; import models.versions.Item; import models.versions.VersionHistoryDag; +import org.neo4j.driver.v1.Record; +import util.ElasticSearch; import java.util.ArrayList; import java.util.List; @@ -59,9 +59,11 @@ public Neo4jItemFactory(Neo4jClient dbClient, * @param tags the tags associated with the item * @throws GroundDbException an error inserting data into the database */ - public void insertIntoDatabase(long id, Map tags) throws GroundDbException { + + public void insertIntoDatabase(long id, Map tags) throws GroundException { for (String key : tags.keySet()) { Tag tag = tags.get(key); + ElasticSearch.insertElasticSearch(tag, "item"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id)); diff --git a/app/dao/versions/postgres/PostgresItemFactory.java b/app/dao/versions/postgres/PostgresItemFactory.java index d36aa5aa..6a92eac9 100644 --- a/app/dao/versions/postgres/PostgresItemFactory.java +++ b/app/dao/versions/postgres/PostgresItemFactory.java @@ -25,14 +25,14 @@ import models.versions.GroundType; import models.versions.Item; import models.versions.VersionHistoryDag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import util.ElasticSearch; import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public abstract class PostgresItemFactory implements ItemFactory { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresItemFactory.class); @@ -70,6 +70,7 @@ public void insertIntoDatabase(long id, Map tags) throws GroundExce for (String key : tags.keySet()) { Tag tag = tags.get(key); + ElasticSearch.insertElasticSearch(tag, "item"); List tagInsertion = new ArrayList<>(); tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id)); diff --git a/app/exceptions/GroundElasticSearchException.java b/app/exceptions/GroundElasticSearchException.java new file mode 100644 index 00000000..0eeaba0d --- /dev/null +++ b/app/exceptions/GroundElasticSearchException.java @@ -0,0 +1,9 @@ +package exceptions; + + +public class GroundElasticSearchException extends GroundException { + + public GroundElasticSearchException(String message) { + super(message); + } +} diff --git a/app/util/CassandraFactories.java b/app/util/CassandraFactories.java index 033469f3..828e28d6 100644 --- a/app/util/CassandraFactories.java +++ b/app/util/CassandraFactories.java @@ -12,27 +12,11 @@ * limitations under the License. */ + package util; -import com.google.common.annotations.VisibleForTesting; -import dao.models.EdgeFactory; -import dao.models.EdgeVersionFactory; -import dao.models.GraphFactory; -import dao.models.GraphVersionFactory; -import dao.models.NodeFactory; -import dao.models.NodeVersionFactory; -import dao.models.StructureFactory; -import dao.models.StructureVersionFactory; -import dao.models.cassandra.CassandraEdgeFactory; -import dao.models.cassandra.CassandraEdgeVersionFactory; -import dao.models.cassandra.CassandraGraphFactory; -import dao.models.cassandra.CassandraGraphVersionFactory; -import dao.models.cassandra.CassandraNodeFactory; -import dao.models.cassandra.CassandraNodeVersionFactory; -import dao.models.cassandra.CassandraRichVersionFactory; -import dao.models.cassandra.CassandraStructureFactory; -import dao.models.cassandra.CassandraStructureVersionFactory; -import dao.models.cassandra.CassandraTagFactory; +import dao.models.*; +import dao.models.cassandra.*; import dao.usage.LineageEdgeFactory; import dao.usage.LineageEdgeVersionFactory; import dao.usage.LineageGraphFactory; @@ -41,21 +25,19 @@ import dao.usage.cassandra.CassandraLineageEdgeVersionFactory; import dao.usage.cassandra.CassandraLineageGraphFactory; import dao.usage.cassandra.CassandraLineageGraphVersionFactory; -import dao.versions.cassandra.CassandraItemFactory; -import dao.versions.cassandra.CassandraVersionFactory; import dao.versions.cassandra.CassandraVersionHistoryDagFactory; import dao.versions.cassandra.CassandraVersionSuccessorFactory; import db.CassandraClient; import db.DbClient; import exceptions.GroundDbException; +import play.Configuration; + import javax.inject.Inject; import javax.inject.Singleton; -import play.Configuration; @Singleton public class CassandraFactories implements FactoryGenerator { private final CassandraClient cassandraClient; - private final CassandraStructureFactory structureFactory; private final CassandraStructureVersionFactory structureVersionFactory; private final CassandraEdgeFactory edgeFactory; @@ -64,7 +46,7 @@ public class CassandraFactories implements FactoryGenerator { private final CassandraGraphVersionFactory graphVersionFactory; private final CassandraNodeFactory nodeFactory; private final CassandraNodeVersionFactory nodeVersionFactory; - + private final CassandraTagFactory tagFactory; private final CassandraLineageEdgeFactory lineageEdgeFactory; private final CassandraLineageEdgeVersionFactory lineageEdgeVersionFactory; private final CassandraLineageGraphFactory lineageGraphFactory; @@ -95,7 +77,7 @@ public CassandraFactories(Configuration configuration) throws GroundDbException new CassandraVersionSuccessorFactory(cassandraClient, idGenerator); CassandraVersionHistoryDagFactory versionHistoryDagFactory = new CassandraVersionHistoryDagFactory(cassandraClient, versionSuccessorFactory); - CassandraTagFactory tagFactory = new CassandraTagFactory(cassandraClient); + CassandraTagFactory tagFactory = new CassandraTagFactory(cassandraClient, dbConf.getBoolean("elasticSearchOn")); this.structureFactory = new CassandraStructureFactory(cassandraClient, versionHistoryDagFactory, tagFactory, idGenerator); @@ -124,6 +106,7 @@ public CassandraFactories(Configuration configuration) throws GroundDbException versionHistoryDagFactory, tagFactory, idGenerator); this.lineageGraphVersionFactory = new CassandraLineageGraphVersionFactory(cassandraClient, this.lineageGraphFactory, this.structureVersionFactory, tagFactory, idGenerator); + this.tagFactory = tagFactory; } @Override @@ -186,7 +169,11 @@ public LineageGraphVersionFactory getLineageGraphVersionFactory() { return this.lineageGraphVersionFactory; } - @Override + + public TagFactory getTagFactory() { + return this.tagFactory; + } + public DbClient getDbClient() { return this.cassandraClient; } diff --git a/app/util/ElasticSearch.java b/app/util/ElasticSearch.java new file mode 100644 index 00000000..f44646eb --- /dev/null +++ b/app/util/ElasticSearch.java @@ -0,0 +1,78 @@ +package util; + + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import exceptions.GroundElasticSearchException; +import models.models.Tag; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.node.Node; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import static org.elasticsearch.node.NodeBuilder.nodeBuilder; + +public class ElasticSearch { + private static Node node; + private static Client client; + private static String clusterName = "groundv1"; + + + + public static boolean connectElasticSearch() throws GroundElasticSearchException { + try { + node = nodeBuilder().clusterName(clusterName).node(); + client = node.client(); + } catch (Exception e) { + throw new GroundElasticSearchException("ElasticSearch encountered an error while trying to connect"); + } + return !node.isClosed(); + } + + public static void insertElasticSearch(Tag tag, String table) throws GroundElasticSearchException { + ObjectMapper mapper = new ObjectMapper(); + try { + String json = mapper.writeValueAsString(tag); + IndexResponse response = client.prepareIndex(clusterName, table, Long.toString(tag.getId())) + .setSource(json).get(); + + } catch (JsonProcessingException e) { + + throw new GroundElasticSearchException("ObjectMapper failed to parse Tag object"); + } + } + + public static List getSearchResponse(String type, String searchQuery) throws GroundElasticSearchException { + client.admin().indices().prepareRefresh().execute().actionGet(); // need to refresh index with new inserted item + SearchResponse response = client.prepareSearch().setTypes(type).setQuery(QueryBuilders.matchQuery("key", searchQuery)).get(); + SearchHit[] hits = response.getHits().hits(); + ObjectMapper mapper = new ObjectMapper(); + List tagIds = new ArrayList<>(); + try { + for (SearchHit hit : hits) { + String source = hit.getSourceAsString(); + Tag tag = mapper.readValue(source, Tag.class); + tagIds.add(tag.getId()); + + } + } catch (JsonParseException e) { + throw new GroundElasticSearchException("ObjectMapper failed to parse json string into Tag object"); + } catch (JsonMappingException e) { + throw new GroundElasticSearchException("ObjectMapper failed to map json string into Tag object"); + } catch (IOException e) { + throw new GroundElasticSearchException("ObjectMapper detected an IOException"); + } + return tagIds; + } + + public static void closeElasticSearch() throws IOException { + node.close(); + client.close(); + } +} diff --git a/app/util/FactoryGenerator.java b/app/util/FactoryGenerator.java index 8807e71a..abebc298 100644 --- a/app/util/FactoryGenerator.java +++ b/app/util/FactoryGenerator.java @@ -12,22 +12,17 @@ * limitations under the License. */ + package util; -import dao.models.EdgeFactory; -import dao.models.EdgeVersionFactory; -import dao.models.GraphFactory; -import dao.models.GraphVersionFactory; -import dao.models.NodeFactory; -import dao.models.NodeVersionFactory; -import dao.models.StructureFactory; -import dao.models.StructureVersionFactory; +import dao.models.*; import dao.usage.LineageEdgeFactory; import dao.usage.LineageEdgeVersionFactory; import dao.usage.LineageGraphFactory; import dao.usage.LineageGraphVersionFactory; import db.DbClient; + public interface FactoryGenerator { EdgeFactory getEdgeFactory(); @@ -53,5 +48,7 @@ public interface FactoryGenerator { LineageGraphVersionFactory getLineageGraphVersionFactory(); + TagFactory getTagFactory(); + DbClient getDbClient(); } diff --git a/app/util/Neo4jFactories.java b/app/util/Neo4jFactories.java index 48fab240..3931202a 100644 --- a/app/util/Neo4jFactories.java +++ b/app/util/Neo4jFactories.java @@ -14,24 +14,8 @@ package util; -import dao.models.EdgeFactory; -import dao.models.EdgeVersionFactory; -import dao.models.GraphFactory; -import dao.models.GraphVersionFactory; -import dao.models.NodeFactory; -import dao.models.NodeVersionFactory; -import dao.models.StructureFactory; -import dao.models.StructureVersionFactory; -import dao.models.neo4j.Neo4jEdgeFactory; -import dao.models.neo4j.Neo4jEdgeVersionFactory; -import dao.models.neo4j.Neo4jGraphFactory; -import dao.models.neo4j.Neo4jGraphVersionFactory; -import dao.models.neo4j.Neo4jNodeFactory; -import dao.models.neo4j.Neo4jNodeVersionFactory; -import dao.models.neo4j.Neo4jRichVersionFactory; -import dao.models.neo4j.Neo4jStructureFactory; -import dao.models.neo4j.Neo4jStructureVersionFactory; -import dao.models.neo4j.Neo4jTagFactory; +import dao.models.*; +import dao.models.neo4j.*; import dao.usage.LineageEdgeFactory; import dao.usage.LineageEdgeVersionFactory; import dao.usage.LineageGraphFactory; @@ -40,15 +24,15 @@ import dao.usage.neo4j.Neo4jLineageEdgeVersionFactory; import dao.usage.neo4j.Neo4jLineageGraphFactory; import dao.usage.neo4j.Neo4jLineageGraphVersionFactory; -import dao.versions.neo4j.Neo4jItemFactory; import dao.versions.neo4j.Neo4jVersionHistoryDagFactory; import dao.versions.neo4j.Neo4jVersionSuccessorFactory; import db.DbClient; import db.Neo4jClient; import exceptions.GroundDbException; +import play.Configuration; + import javax.inject.Inject; import javax.inject.Singleton; -import play.Configuration; @Singleton public class Neo4jFactories implements FactoryGenerator { @@ -62,12 +46,13 @@ public class Neo4jFactories implements FactoryGenerator { private final Neo4jGraphVersionFactory graphVersionFactory; private final Neo4jNodeFactory nodeFactory; private final Neo4jNodeVersionFactory nodeVersionFactory; - + private final Neo4jTagFactory tagFactory; private final Neo4jLineageEdgeFactory lineageEdgeFactory; private final Neo4jLineageEdgeVersionFactory lineageEdgeVersionFactory; private final Neo4jLineageGraphFactory lineageGraphFactory; private final Neo4jLineageGraphVersionFactory lineageGraphVersionFactory; + /** * Create the Neo4j factories. * @@ -92,7 +77,7 @@ public Neo4jFactories(Configuration configuration) throws GroundDbException { new Neo4jVersionSuccessorFactory(this.neo4jClient, idGenerator); Neo4jVersionHistoryDagFactory versionHistoryDagFactory = new Neo4jVersionHistoryDagFactory(this.neo4jClient, versionSuccessorFactory); - Neo4jTagFactory tagFactory = new Neo4jTagFactory(this.neo4jClient); + Neo4jTagFactory tagFactory = new Neo4jTagFactory(this.neo4jClient, dbConf.getBoolean("elasticSearchOn")); this.structureFactory = new Neo4jStructureFactory(this.neo4jClient, versionHistoryDagFactory, tagFactory, idGenerator); @@ -106,6 +91,7 @@ public Neo4jFactories(Configuration configuration) throws GroundDbException { this.graphFactory = new Neo4jGraphFactory(this.neo4jClient, versionHistoryDagFactory, tagFactory, idGenerator); + this.graphVersionFactory = new Neo4jGraphVersionFactory(this.neo4jClient, this.graphFactory, this.structureVersionFactory, tagFactory, idGenerator); this.nodeFactory = new Neo4jNodeFactory(this.neo4jClient, versionHistoryDagFactory, @@ -121,6 +107,7 @@ public Neo4jFactories(Configuration configuration) throws GroundDbException { versionHistoryDagFactory, tagFactory, idGenerator); this.lineageGraphVersionFactory = new Neo4jLineageGraphVersionFactory(this.neo4jClient, this.lineageGraphFactory, this.structureVersionFactory, tagFactory, idGenerator); + this.tagFactory = tagFactory; } @Override @@ -184,6 +171,10 @@ public LineageGraphVersionFactory getLineageGraphVersionFactory() { } @Override + public TagFactory getTagFactory() { + return this.tagFactory; + } + public DbClient getDbClient() { return this.neo4jClient; } diff --git a/app/util/PostgresFactories.java b/app/util/PostgresFactories.java index 43c34be7..b026f9f9 100644 --- a/app/util/PostgresFactories.java +++ b/app/util/PostgresFactories.java @@ -12,26 +12,11 @@ * limitations under the License. */ + package util; -import dao.models.EdgeFactory; -import dao.models.EdgeVersionFactory; -import dao.models.GraphFactory; -import dao.models.GraphVersionFactory; -import dao.models.NodeFactory; -import dao.models.NodeVersionFactory; -import dao.models.StructureFactory; -import dao.models.StructureVersionFactory; -import dao.models.postgres.PostgresEdgeFactory; -import dao.models.postgres.PostgresEdgeVersionFactory; -import dao.models.postgres.PostgresGraphFactory; -import dao.models.postgres.PostgresGraphVersionFactory; -import dao.models.postgres.PostgresNodeFactory; -import dao.models.postgres.PostgresNodeVersionFactory; -import dao.models.postgres.PostgresRichVersionFactory; -import dao.models.postgres.PostgresStructureFactory; -import dao.models.postgres.PostgresStructureVersionFactory; -import dao.models.postgres.PostgresTagFactory; +import dao.models.*; +import dao.models.postgres.*; import dao.usage.LineageEdgeFactory; import dao.usage.LineageEdgeVersionFactory; import dao.usage.LineageGraphFactory; @@ -40,16 +25,15 @@ import dao.usage.postgres.PostgresLineageEdgeVersionFactory; import dao.usage.postgres.PostgresLineageGraphFactory; import dao.usage.postgres.PostgresLineageGraphVersionFactory; -import dao.versions.postgres.PostgresItemFactory; -import dao.versions.postgres.PostgresVersionFactory; import dao.versions.postgres.PostgresVersionHistoryDagFactory; import dao.versions.postgres.PostgresVersionSuccessorFactory; import db.DbClient; import db.PostgresClient; import exceptions.GroundDbException; +import play.Configuration; + import javax.inject.Inject; import javax.inject.Singleton; -import play.Configuration; @Singleton public class PostgresFactories implements FactoryGenerator { @@ -68,6 +52,7 @@ public class PostgresFactories implements FactoryGenerator { private final PostgresLineageEdgeVersionFactory lineageEdgeVersionFactory; private final PostgresLineageGraphFactory lineageGraphFactory; private final PostgresLineageGraphVersionFactory lineageGraphVersionFactory; + private final PostgresTagFactory tagFactory; /** * Create the Postgres factories. @@ -94,7 +79,7 @@ public PostgresFactories(Configuration configuration) throws GroundDbException { new PostgresVersionSuccessorFactory(this.postgresClient, idGenerator); PostgresVersionHistoryDagFactory versionHistoryDagFactory = new PostgresVersionHistoryDagFactory(this.postgresClient, versionSuccessorFactory); - PostgresTagFactory tagFactory = new PostgresTagFactory(this.postgresClient); + PostgresTagFactory tagFactory = new PostgresTagFactory(this.postgresClient, dbConf.getBoolean("elasticSearchOn")); this.structureFactory = new PostgresStructureFactory(this.postgresClient, versionHistoryDagFactory, tagFactory, idGenerator); @@ -123,6 +108,7 @@ public PostgresFactories(Configuration configuration) throws GroundDbException { versionHistoryDagFactory, tagFactory, idGenerator); this.lineageGraphVersionFactory = new PostgresLineageGraphVersionFactory(this.postgresClient, this.lineageGraphFactory, this.structureVersionFactory, tagFactory, idGenerator); + this.tagFactory = tagFactory; } @Override @@ -186,6 +172,10 @@ public LineageGraphVersionFactory getLineageGraphVersionFactory() { } @Override + public TagFactory getTagFactory() { + return this.tagFactory; + } + public DbClient getDbClient() { return this.postgresClient; } diff --git a/build.sbt b/build.sbt index 7f33898e..dfbd51bb 100644 --- a/build.sbt +++ b/build.sbt @@ -15,6 +15,8 @@ libraryDependencies += "org.postgresql" % "postgresql" % "9.4.1208" libraryDependencies += "org.neo4j.driver" % "neo4j-java-driver" % "1.2.0" libraryDependencies += "org.mockito" % "mockito-core" % "2.7.22" % "test" libraryDependencies += "org.assertj" % "assertj-core" % "3.6.2" % "test" +libraryDependencies += "org.elasticsearch" % "elasticsearch" % "1.5.1" + // disable parallel execution of tests parallelExecution in Test := false diff --git a/conf/application.conf b/conf/application.conf index 2c3fa75d..10eb5b40 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -7,6 +7,7 @@ db.host=localhost db.port=5432 db.user=ground db.password=metadata +db.elasticSearchOn=true machine.count=1 machine.id=0 diff --git a/test/dao/CassandraTest.java b/test/dao/CassandraTest.java index a1d5f544..e16314e2 100644 --- a/test/dao/CassandraTest.java +++ b/test/dao/CassandraTest.java @@ -14,25 +14,24 @@ package dao; -import java.util.HashMap; -import java.util.Map; -import play.Configuration; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.util.function.Function; - import dao.models.cassandra.CassandraStructureVersionFactory; import dao.models.cassandra.CassandraTagFactory; import dao.versions.cassandra.CassandraVersionHistoryDagFactory; import dao.versions.cassandra.CassandraVersionSuccessorFactory; import db.CassandraClient; import exceptions.GroundDbException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import play.Configuration; import util.CassandraFactories; import util.IdGenerator; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + public class CassandraTest extends DaoTest { private static final String TRUNCATE_SCRIPT = "./scripts/cassandra/truncate.cql"; @@ -59,7 +58,7 @@ public static void setup() throws GroundDbException { new IdGenerator(0, 1, false)); versionHistoryDAGFactory = new CassandraVersionHistoryDagFactory(cassandraClient, versionSuccessorFactory); - tagFactory = new CassandraTagFactory(cassandraClient); + tagFactory = new CassandraTagFactory(cassandraClient, true); edgeFactory = factories.getEdgeFactory(); graphFactory = factories.getGraphFactory(); diff --git a/test/dao/Neo4jTest.java b/test/dao/Neo4jTest.java index a661311c..95c802ec 100644 --- a/test/dao/Neo4jTest.java +++ b/test/dao/Neo4jTest.java @@ -14,32 +14,32 @@ package dao; -import com.typesafe.config.Config; -import exceptions.GroundException; -import java.util.HashMap; -import java.util.Map; -import play.Configuration; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; import dao.models.neo4j.Neo4jStructureVersionFactory; import dao.models.neo4j.Neo4jTagFactory; import dao.versions.neo4j.Neo4jVersionHistoryDagFactory; import dao.versions.neo4j.Neo4jVersionSuccessorFactory; import db.Neo4jClient; +import exceptions.GroundException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import play.Configuration; +import util.ElasticSearch; import util.IdGenerator; import util.Neo4jFactories; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + + public class Neo4jTest extends DaoTest { /* Note: In Neo4j, we don't create explicit (Rich)Versions because all of the logic is wrapped in * FooVersions. We are using NodeVersions as stand-ins because they are the most simple kind of * Versions. */ private static Neo4jFactories factories; - protected static Neo4jClient neo4jClient; protected static Neo4jVersionSuccessorFactory versionSuccessorFactory; protected static Neo4jVersionHistoryDagFactory versionHistoryDAGFactory; @@ -52,7 +52,7 @@ public static void setupClass() throws GroundException { versionSuccessorFactory = new Neo4jVersionSuccessorFactory(neo4jClient, new IdGenerator(0, 1, true)); versionHistoryDAGFactory = new Neo4jVersionHistoryDagFactory(neo4jClient, versionSuccessorFactory); - tagFactory = new Neo4jTagFactory(neo4jClient); + tagFactory = new Neo4jTagFactory(neo4jClient, true); edgeFactory = factories.getEdgeFactory(); @@ -84,6 +84,12 @@ public void setup() throws IOException, InterruptedException { neo4jClient.dropData(); } + + @AfterClass + public static void tearDown() throws IOException { + ElasticSearch.closeElasticSearch(); + } + private static Configuration createTestConfig() { Map confMap = new HashMap<>(); Map dbMap = new HashMap<>(); diff --git a/test/dao/PostgresTest.java b/test/dao/PostgresTest.java index 12506f5e..5dfc6751 100644 --- a/test/dao/PostgresTest.java +++ b/test/dao/PostgresTest.java @@ -14,27 +14,26 @@ package dao; -import java.util.HashMap; -import java.util.Map; -import play.Configuration; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; - import dao.models.postgres.PostgresStructureVersionFactory; import dao.models.postgres.PostgresTagFactory; import dao.versions.postgres.PostgresVersionHistoryDagFactory; import dao.versions.postgres.PostgresVersionSuccessorFactory; import db.PostgresClient; import exceptions.GroundDbException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import play.Configuration; import util.IdGenerator; import util.PostgresFactories; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + public class PostgresTest extends DaoTest { private static final String DROP_SCRIPT = "./scripts/postgres/drop_postgres.sql"; private static final String CREATE_SCHEMA_SCRIPT = "./scripts/postgres/postgres.sql"; @@ -52,7 +51,7 @@ public static void setupClass() throws GroundDbException { versionSuccessorFactory = new PostgresVersionSuccessorFactory(postgresClient, new IdGenerator(0, 1, false)); versionHistoryDAGFactory = new PostgresVersionHistoryDagFactory(postgresClient, versionSuccessorFactory); - tagFactory = new PostgresTagFactory(postgresClient); + tagFactory = new PostgresTagFactory(postgresClient, true); edgeFactory = factories.getEdgeFactory(); graphFactory = factories.getGraphFactory(); diff --git a/test/dao/models/cassandra/CassandraTagFactoryTest.java b/test/dao/models/cassandra/CassandraTagFactoryTest.java index 00817506..28e34a24 100644 --- a/test/dao/models/cassandra/CassandraTagFactoryTest.java +++ b/test/dao/models/cassandra/CassandraTagFactoryTest.java @@ -14,6 +14,10 @@ package dao.models.cassandra; +import dao.CassandraTest; +import exceptions.GroundException; +import models.models.Tag; +import models.versions.GroundType; import org.junit.Test; import java.util.ArrayList; @@ -21,12 +25,7 @@ import java.util.List; import java.util.Map; -import dao.CassandraTest; -import models.models.Tag; -import models.versions.GroundType; -import exceptions.GroundException; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class CassandraTagFactoryTest extends CassandraTest { @@ -37,12 +36,13 @@ public CassandraTagFactoryTest() throws GroundException { @Test public void testGetItemIdsByTag() throws GroundException { Map tagsMap = new HashMap<>(); - tagsMap.put("testtag", new Tag(1, "testtag", "tag", GroundType.STRING)); + Tag tag = new Tag(1, "testtag", "tag", GroundType.STRING); + tagsMap.put("testtag", tag); long nodeId1 = CassandraTest.nodeFactory.create(null, "test1", tagsMap).getId(); long nodeId2 = CassandraTest.nodeFactory.create(null, "test2", tagsMap).getId(); - List ids = CassandraTest.tagFactory.getItemIdsByTag("testtag"); + List ids = CassandraTest.tagFactory.getItemIdsByTag(tag.getKey()); CassandraTest.cassandraClient.commit(); @@ -53,7 +53,8 @@ public void testGetItemIdsByTag() throws GroundException { @Test public void testGetVersionIdsByTag() throws GroundException { Map tagsMap = new HashMap<>(); - tagsMap.put("testtag", new Tag(1, "testtag", "tag", GroundType.STRING)); + Tag tag = new Tag(1, "testtag", "tag", GroundType.STRING); + tagsMap.put("testtag", tag); long nodeId = CassandraTest.createNode("testNode").getId(); @@ -62,7 +63,7 @@ public void testGetVersionIdsByTag() throws GroundException { long nodeVersionId2 = CassandraTest.nodeVersionFactory.create(tagsMap, -1, null, new HashMap<>(), nodeId, new ArrayList<>()).getId(); - List ids = CassandraTest.tagFactory.getVersionIdsByTag("testtag"); + List ids = CassandraTest.tagFactory.getVersionIdsByTag(tag.getKey()); CassandraTest.cassandraClient.commit();