diff --git a/.gitignore b/.gitignore index 9daa8247..1e700d61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,32 @@ -.DS_Store +# Logs +logs +*.log + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# Deployed apps should consider commenting this line out: +# see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git node_modules + +# Ignore .env configuration files +.env + +# Ignore .DS_Store files on OS X +.DS_Store +.idea diff --git a/.travis.yml b/.travis.yml index 690e2dd2..07b5d92c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,8 @@ language: node_js node_js: - - 0.11 - 0.10 - - 0.8 services: - mongodb - elasticsearch - -notifications: - email: - - james.r.carr@gmail.com diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..6785832b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +## 2.0.0 (2014-10-10) + +Features: + +- Moved to [official elasticsearch driver](https://github.com/elasticsearch/elasticsearch-js) + - Caused `search` api to conform closer to official driver + - Added options to searching +- Refactored bulk api +- Refreshed README.md +- Added CHANGELOG.md +- Added CONTRIBUTING.md +- Added LICENSE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..7d8e095c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing +Pull requests are always welcome as long as an accompanying test case is +associated. + +This project is configured to use [git +flow](https://github.com/nvie/gitflow/) and the following conventions +are used: + +* ``develop`` - represents current active development and can possibly be + unstable. +* ``master`` - pristine copy of repository, represents the currently + stable release found in the npm index. +* ``feature/**`` - represents a new feature being worked on + +If you wish to contribute, the only requirement is to: + +- branch a new feature branch from develop (if you're working on an + issue, prefix it with the issue number) +- make the changes, with accompanying test cases +- issue a pull request against develop branch + +Although I use git flow and prefix feature branches with "feature/" I +don't require this for pull requests... all I care is that the feature +branch name makes sense. + +Pulls requests against master or pull requests branched from master will +be rejected. + +## Examples +Someone picks up issue #39 on selective indexing. + +Good branch names: +* 39-selective-indexing +* feature/39-selective-indexing + +Someone submits a new feature that allows shard configuration: + +Good branch names: +* feature/shard-configuration +* shard-configuration +* or file an issue, then create a feature branch + +Feel free to ping me if you need help! :) + +## Running Tests +In order to run the tests you will need: + +* An elasticsearch server running on port 9200 +* A mongodb server +* [mocha](http://visionmedia.github.com/mocha/) + +With those installed, running ''npm test'' will run the tests with the +preferred timeout (which is extended for integration tests. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..61673abd --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,9 @@ +[The MIT License](https://tldrlegal.com/l/mit) + +Copyright (c) 2012 James R. Carr + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/readme.md b/README.md similarity index 54% rename from readme.md rename to README.md index 22fb8b1a..8f8d5e96 100644 --- a/readme.md +++ b/README.md @@ -1,30 +1,54 @@ # Mongoosastic [![Build -Status](https://secure.travis-ci.org/jamescarr/mongoosastic.png?branch=master)](http://travis-ci.org/jamescarr/mongoosastic) - -A [mongoose](http://mongoosejs.com/) plugin that indexes models into [elasticsearch](http://www.elasticsearch.org/). I kept -running into cases where I needed full text search capabilities in my -mongodb based models only to discover mongodb has none. In addition to -full text search, I also needed the ability to filter ranges of data -points in the searches and even highlight matches. For these reasons, -elastic search was a perfect fit and hence this project. - -## Current Version -The current version is ``0.6.0`` +Status](https://secure.travis-ci.org/mongoosastic/mongoosastic.png?branch=master)](http://travis-ci.org/mongoosastic/mongoosastic) +[![NPM version](https://badge.fury.io/js/mongoosastic.svg)](http://badge.fury.io/js/mongoosastic) +[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/mongoosastic/mongoosastic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +Mongoosastic is a [mongoose](http://mongoosejs.com/) plugin that can automatically index your models into [elasticsearch](http://www.elasticsearch.org/). + +- [Installation](#installation) +- [Setup](#setup) +- [Indexing](#indexing) + - [Saving a document](#saving-a-document) + - [Indexing nested models](#indexing-nested-models) + - [Indexing an existing collection](#indexing-an-existing-collection) + - [Bulk indexing](#bulk-indexing) + - [Indexing on demand](#indexing-on-demand) + - [Truncating an index](#truncating-an-index) +- [Mapping](#mapping) + - [Geo mapping](#geo-mapping) + - [Indexing a geo point](#indexing-a-geo-point) + - [Indexing a geo shape](#indexing-a-geo-shape) + - [Creating mappings on-demand](#creating-mappings-on-demand) +- [Queries](#queries) + - [Hydration](#hydration) ## Installation ```bash -npm install mongoosastic - +npm install -S mongoosastic ``` -Or add it to your package.json +## Setup + +### Model.plugin(mongoosastic, options) + +Options are: -## Usage +* `index` - the index in elastic search to use. Defaults to the + pluralization of the model name. +* `type` - the type this model represents in elastic search. Defaults + to the model name. +* `host` - the host elastic search is running on +* `port` - the port elastic search is running on +* `auth` - the authentication needed to reach elastic search server. In the standard format of 'username:password' +* `protocol` - the protocol the elastic search server uses. Defaults to http +* `hydrate` - whether or not to lookup results in mongodb before +* `hydrateOptions` - options to pass into hydrate function +* `bulk` - size and delay options for bulk indexing -To make a model indexed into elastic search simply add the plugin. +To have a model indexed into elastic search simply add the plugin. ```javascript var mongoose = require('mongoose') @@ -36,8 +60,12 @@ var User = new Schema({ , email: String , city: String }) +var options = { + host:"localhost", + port:9200 +}; -User.plugin(mongoosastic) +User.plugin(mongoosastic.plugin(options)) ``` This will by default simply use the pluralization of the model name as the index @@ -59,13 +87,46 @@ var User = new Schema({ , city: String }) -User.plugin(mongoosastic) +User.plugin(mongoosastic.plugin()) ``` In this case only the name field will be indexed for searching. -####Indexing Nested Models +Now, by adding the plugin, the model will have a new method called +`search` which can be used to make simple to complex searches. The `search` +method accepts [standard elasticsearch query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-queries.html) + +```javascript +User.search({ + query_string: { + query: "john" + } +}, function(err, results) { + // results here +}); + +``` + +## Indexing + +### Saving a document +The indexing takes place after saving inside the mongodb and is a defered process. +One can check the end of the indexion catching es-indexed event. + +```javascript +doc.save(function(err){ + if (err) throw err; + /* Document indexation on going */ + doc.on('es-indexed', function(err, res){ + if (err) throw err; + /* Document is indexed */ + }); + }); +``` + + +###Indexing Nested Models In order to index nested models you can refer following example. ```javascript @@ -83,19 +144,9 @@ var User = new Schema({ , comments: {type:[Comment], es_indexed:true} }) -User.plugin(mongoosastic) +User.plugin(mongoosastic.plugin()) ``` -Finally, adding the plugin will add a new method to the model called -search which can be used to make simple to complex searches. - -```javascript - -User.search({query:"john"}, function(err, results) { - // results here -}); - -``` ### Indexing An Existing Collection Already have a mongodb collection that you'd like to index using this @@ -129,9 +180,53 @@ You can also synchronize a subset of documents based on a query! var stream = Book.synchronize({author: 'Arthur C. Clarke'}) ``` -One caveat... synchronization is kinda slow for now. Use with care. +### Bulk Indexing + +You can also specify `bulk` options with mongoose which will utilize elasticsearch's bulk indexing api. This will cause the `synchronize` function to use bulk indexing as well. + +Mongoosastic will wait 1 second (or specified delay) until it has 1000 docs (or specified size) and then perform bulk indexing. + +```javascript +BookSchema.plugin(mongoosastic.plugin(), { + bulk: { + size: 10, // preferred number of docs to bulk index + delay: 100 //milliseconds to wait for enough docs to meet size constraint + } +}); +``` + +### Indexing On Demand +You can do on-demand indexes using the `index` function + +```javascript +Dude.findOne({name:'Jeffery Lebowski', function(err, dude){ + dude.awesome = true; + dude.index(function(err, res){ + console.log("egads! I've been indexed!"); + }); +}); +``` + +The index method takes 2 arguments: + +* `options` (optional) - {index, type} - the index and type to publish to. Defaults to the standard index and type. + the model was setup with. +* `callback` - callback function to be invoked when model has been + indexed. + +Note that indexing a model does not mean it will be persisted to +mongodb. Use save for that. + +### Truncating an index + +The static method `esTruncate` will delete all documents from the associated index. This method combined with synchronise can be usefull in case of integration tests for example when each test case needs a cleaned up index in ElasticSearch. + +```javascript +GarbageModel.esTruncate(function(err){...}); +``` + +## Mapping -### Per Field Options Schemas can be configured to have special options per field. These match with the existing [field mapping configurations](http://www.elasticsearch.org/guide/reference/mapping/core-types.html) defined by elasticsearch with the only difference being they are all prefixed by "es_". @@ -151,46 +246,7 @@ This example uses a few other mapping fields... such as null_value and type (which overrides whatever value the schema type is, useful if you want stronger typing such as float). -#### Creating Mappings for These Features -The way this can be mapped in elastic search is by creating a mapping -for the index the model belongs to. Currently to the best of my -knowledge mappings are create once when creating an index and can only -be modified by destroying the index. The optionnal first parameter is -the settings option for the index (for defining analysers for example or whatever is [there](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html). - -As such, creating the mapping is a one time operation and can be done as -follows (using the BookSchema as an example): - -```javascript -var BookSchema = new Schema({ - title: {type:String, es_boost:2.0} - , author: {type:String, es_null_value:"Unknown Author"} - , publicationDate: {type:Date, es_type:'date'} - -BookSchema.plugin(mongoosastic); -var Book = mongoose.model('Book', BookSchema); -Book.createMapping({ - "analysis" : { - "analyzer":{ - "content":{ - "type":"custom", - "tokenizer":"whitespace" - } - } - } -},function(err, mapping){ - // do neat things here -}); - -``` -This feature is still a work in progress. As of this writing you'll have -to manage whether or not you need to create the mapping, mongoosastic -will make no assumptions and simply attempt to create the mapping. If -the mapping already exists, an Exception detailing such will be -populated in the `err` argument. - -#### Mapping options -There are various types that can be defined in elasticsearch. Check out http://www.elasticsearch.org/guide/reference/mapping/ for more information. Here are examples to the currently possible definitions in mongoosastic: +There are various mapping options that can be defined in elasticsearch. Check out [http://www.elasticsearch.org/guide/reference/mapping/](http://www.elasticsearch.org/guide/reference/mapping/) for more information. Here are examples to the currently possible definitions in mongoosastic: ```javascript var ExampleSchema = new Schema({ @@ -242,6 +298,17 @@ var ExampleSchema = new Schema({ lon: { type: Number } } + geo_shape: { + coordinates : [], + type: {type: String}, + geo_shape: { + type:String, + es_type: "geo_shape", + es_tree: "quadtree", + es_precision: "1km" + } + } + // Special feature : specify a cast method to pre-process the field before indexing it someFieldToCast : { type: String, @@ -249,6 +316,7 @@ var ExampleSchema = new Schema({ return value + ' something added'; } } + }); // Used as nested schema above. @@ -258,19 +326,104 @@ var SubSchema = new Schema({ }); ``` -### Advanced Queries +### Geo mapping +Prior to index any geo mapped data (or calling the synchronize), +the mapping must be manualy created with the createMapping (see above). + +Notice that the name of the field containing the ES geo data must start by +'geo_' to be recognize as such. + +#### Indexing a geo point + +```javascript +var geo = new GeoModel({ + /* … */ + geo_with_lat_lon: { lat: 1, lon: 2} + /* … */ +}); +``` + +#### Indexing a geo shape + +```javascript +var geo = new GeoModel({ + … + geo_shape:{ + type:'envelope', + coordinates: [[3,4],[1,2] /* Arrays of coord : [[lon,lat],[lon,lat]] */ + } + … +}); +``` + +Mapping, indexing and searching example for geo shape can be found in test/geo-test.js + +For example, one can retrieve the list of document where the shape contain a specific +point (or polygon...) + +```javascript +var geoQuery = { + "match_all": {} + } + +var geoFilter = { + geo_shape: { + geo_shape": { + shape: { + type: "point", + coordinates: [3,1] + } + } + } + } + +GeoModel.search(geoQuery, {filter: geoFilter}, function(err, res) { /* ... */ }) +``` + +### Creating Mappings On Demand +Creating the mapping is a one time operation and can be done as +follows (using the BookSchema as an example): + +```javascript +var BookSchema = new Schema({ + title: {type:String, es_boost:2.0} + , author: {type:String, es_null_value:"Unknown Author"} + , publicationDate: {type:Date, es_type:'date'} + +BookSchema.plugin(mongoosastic); +var Book = mongoose.model('Book', BookSchema); +Book.createMapping({ + "analysis" : { + "analyzer":{ + "content":{ + "type":"custom", + "tokenizer":"whitespace" + } + } + } +},function(err, mapping){ + // do neat things here +}); + +``` +This feature is still a work in progress. As of this writing you'll have +to manage whether or not you need to create the mapping, mongoosastic +will make no assumptions and simply attempt to create the mapping. If +the mapping already exists, an Exception detailing such will be +populated in the `err` argument. + + +## Queries The full query DSL of elasticsearch is exposed through the search method. For example, if you wanted to find all people between ages 21 and 30: ```javascript Person.search({ - query:{ - range: { - age:{ - from:21 - , to: 30 - } + range: { + age:{ + from:21 + , to: 30 } } }, function(err, people){ @@ -278,9 +431,19 @@ Person.search({ }); ``` - See the elasticsearch [Query DSL](http://www.elasticsearch.org/guide/reference/query-dsl/) docs for more information. +You can also specify query options like [sorts](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html#search-request-sort) + +```javascript +Person.search({/* ... */}, {sort: "age:asc"}, function(err, people){ + //sorted results +}); +``` + +Options for queries must adhere to the [javascript elasticsearch driver specs](http://www.elasticsearch.org/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-search). + + ### Hydration By default objects returned from performing a search will be the objects as is in elastic search. This is useful in cases where only what was @@ -292,7 +455,7 @@ provide {hydrate:true} as the second argument to a search call. ```javascript -User.search({query:"john"}, {hydrate:true}, function(err, results) { +User.search({query_string: {query: "john"}}, {hydrate:true}, function(err, results) { // results here }); @@ -303,7 +466,7 @@ how to query for the mongoose object. ```javascript -User.search({query:"john"}, {hydrate:true, hydrateOptions: {select: 'name age'}}, function(err, results) { +User.search({query_string: {query: "john"}}, {hydrate:true, hydrateOptions: {select: 'name age'}}, function(err, results) { // results here }); @@ -324,150 +487,30 @@ var User = new Schema({ , city: String }) -User.plugin(mongoosastic, {hydrate:true, hydrateOptions: {lean: true}}) +User.plugin(mongoosastic.plugin(), {hydrate:true, hydrateOptions: {lean: true}}) ``` -### Indexing On Demand -While developing mongoose I came across a scenario where we needed to be -able to save models (and search them) but a single action would -"publish" those models to be searched from a public site. To address -this I create a new method: `index`. - -#### Usage -Usage is as simple as calling index on an existing model. +###Populating ```javascript -Dude.findOne({name:'Jeffery Lebowski', function(err, dude){ - dude.awesome = true; - dude.index(function(err, res){ - console.log("egads! I've been indexed!"); - }); -}); -``` - -The index method takes 3 arguments: - -* `index` (optional) - the index to publish to. Defaults to the index - the model was setup with. -* `type` (optional) - the type to publish as. Defaults to the type the - model was setup with. -* `callback` - callback function to be invoked when model has been - indexed. - -Note that indexing a model does not mean it will be persisted to -mongodb. Use save for that. - -### Truncating an index - -The static method truncate will deleted all documents from the associated index. This method combined with synchronise can be usefull in case of integration tests for example when each test case needs a cleaned up index in ElasticSearch. - -#### Usage - -```javascript -GarbageModel.truncate(function(err){...}); -``` - -### Model.plugin(mongoosastic, options) - -Options are: - -* `index` - the index in elastic search to use. Defaults to the - pluralization of the model name. -* `type` - the type this model represents in elastic search. Defaults - to the model name. -* `host` - the host elastic search is running on -* `port` - the port elastic search is running on -* `auth` - the authentication needed to reach elastic search server. In the standard format of 'username:password' -* `protocol` - the protocol the elastic search server uses. Defaults to http -* `hydrate` - whether or not to lookup results in mongodb before - returning results from a search. Defaults to false. -* `curlDebug` - elastical debugging. Defaults to false. - -Here are all other avaible options invloved in connection to elastic search server: -https://ramv.github.io/node-elastical/docs/classes/Client.html - -Experimental Options: - -#### Specifying Different Index and Type -Perhaps you have an existing index and you want to specify the index and -type used to index your document? No problem!! - -```javascript -var SupervisorSchema = new Schema({ - name: String -, department: String +var mongoosastic = require('mongoosastic'); + +mongoosastic.search({ + match_all:{} +},{ + index:["articles","videos","musics","gallerys"], + type:["article","video","music","gallery"], + hydrate:true, + hydrateOptions:{ + populate:"tags catgories ..." + } +}, function(err, results) { + if(err){ + console.log(err); + } else { + console.log(results.hits); + } }); -SupervisorSchema.plugin(mongoosastic, {index: 'employees', type:'manager'}); - -var Supervisor = mongoose.model('supervisor', SupervisorSchema); - -``` - -## Contributing -Pull requests are always welcome as long as an accompanying test case is -associated. - -This project is configured to use [git -flow](https://github.com/nvie/gitflow/) and the following conventions -are used: - -* ``develop`` - represents current active development and can possibly be - unstable. -* ``master`` - pristine copy of repository, represents the currently - stable release found in the npm index. -* ``feature/**`` - represents a new feature being worked on - -If you wish to contribute, the only requirement is to: - -- branch a new feature branch from develop (if you're working on an - issue, prefix it with the issue number) -- make the changes, with accompanying test cases -- issue a pull request against develop branch - -Although I use git flow and prefix feature branches with "feature/" I -don't require this for pull requests... all I care is that the feature -branch name makes sense. - -Pulls requests against master or pull requests branched from master will -be rejected. - -#### Examples -Someone picks up issue #39 on selective indexing. - -Good branch names: -* 39-selective-indexing -* feature/39-selective-indexing - -Someone submits a new feature that allows shard configuration: - -Good branch names: -* feature/shard-configuration -* shard-configuration -* or file an issue, then create a feature branch - -Feel free to ping me if you need help! :) - -### Running Tests -In order to run the tests you will need: - -* An elasticsearch server running on port 9200 -* A mongodb server -* [mocha](http://visionmedia.github.com/mocha/) - -With those installed, running ''npm test'' will run the tests with the -preferred timeout (which is extended for integration tests. - - -## License -[The MIT License](https://tldrlegal.com/l/mit) - -Copyright (c) 2012 James R. Carr - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +``` \ No newline at end of file diff --git a/lib/mapping-generator.js b/lib/mapping-generator.js index 1bbf1df4..2950fc0c 100644 --- a/lib/mapping-generator.js +++ b/lib/mapping-generator.js @@ -1,18 +1,15 @@ -function Generator(){ +function Generator() { } -Generator.prototype.generateMapping = function(schema, cb){ - var cleanTree = getCleanTree(schema.tree, schema.paths, ''); - delete cleanTree[schema.get('versionKey')]; - var mapping = getMapping(cleanTree, ''); - - cb(null, { properties: mapping }); +Generator.prototype.generateMapping = function(schema, cb) { + var cleanTree = getCleanTree(schema.tree, schema.paths, ''); + delete cleanTree[schema.get('versionKey')]; + var mapping = getMapping(cleanTree, ''); + cb(null, {properties: mapping}); }; module.exports = Generator; - - // // Generates the mapping // @@ -23,67 +20,65 @@ module.exports = Generator; // @return the mapping // function getMapping(cleanTree, prefix) { - var mapping = {}, - value = {}, - implicitFields = [], - hasEs_index = false; - - if (prefix !== '') { - prefix = prefix + '.'; - } - - for (var field in cleanTree) { - value = cleanTree[field]; - mapping[field] = {}; - mapping[field].type = value.type; - - // Check if field was explicity indexed, if not keep track implicitly - if(value.es_indexed) { - hasEs_index = true; - } else if (value.type) { - implicitFields.push(field); - } - - - // If there is no type, then it's an object with subfields. - if (!value.type) { - mapping[field].type = 'object'; - mapping[field].properties = getMapping(value, prefix + field); - continue; - } - - // If it is a objectid make it a string. - if(value.type === 'objectid'){ - mapping[field].type = 'string'; - continue; - } - - //If indexing a number, and no es_type specified, default to double - if (value.type === 'number' && value['es_type'] === undefined) { - mapping[field].type = 'double'; - continue; - } - - // Else, it has a type and we want to map that! - for (var prop in value) { - // Map to field if it's an Elasticsearch option - if (prop.indexOf('es_') === 0 && prop !== 'es_indexed') { - mapping[field][prop.replace(/^es_/, '')] = value[prop]; - } - } - } - - //If one of the fields was explicitly indexed, delete all implicit fields - if (hasEs_index) { - implicitFields.forEach(function(field) { - delete mapping[field]; - }); - } - - return mapping; + var mapping = {}, + value, + implicitFields = [], + hasEsIndex = false; + + if (prefix !== '') { + prefix = prefix + '.'; + } + + for (var field in cleanTree) { + value = cleanTree[field]; + mapping[field] = {}; + mapping[field].type = value.type; + + // Check if field was explicity indexed, if not keep track implicitly + if (value.es_indexed) { + hasEsIndex = true; + } else if (value.type) { + implicitFields.push(field); + } + + // If there is no type, then it's an object with subfields. + if (!value.type) { + mapping[field].type = 'object'; + mapping[field].properties = getMapping(value, prefix + field); + continue; + } + + // If it is a objectid make it a string. + if (value.type === 'objectid') { + // do not continue here so we can handle other es_ options + mapping[field].type = 'string'; + } + + //If indexing a number, and no es_type specified, default to double + if (value.type === 'number' && value.es_type === undefined) { + mapping[field].type = 'double'; + continue; + } + + // Else, it has a type and we want to map that! + for (var prop in value) { + // Map to field if it's an Elasticsearch option + if (prop.indexOf('es_') === 0 && prop !== 'es_indexed') { + mapping[field][prop.replace(/^es_/, '')] = value[prop]; + } + } + } + + //If one of the fields was explicitly indexed, delete all implicit fields + if (hasEsIndex) { + implicitFields.forEach(function(field) { + delete mapping[field]; + }); + } + + return mapping; } - // // Generates a clean tree // @@ -96,79 +91,92 @@ function getMapping(cleanTree, prefix) { // function getCleanTree(tree, paths, prefix) { - var cleanTree = {}, - type = '', - value = {}; - - if (prefix !== '') { - prefix = prefix + '.'; - } - - for (var field in tree){ - if (prefix === '' && (field === "id" || field === "_id")) { - continue; - } - - type = getTypeFromPaths(paths, prefix + field); - value = tree[field]; - - if(value.es_indexed === false) { - continue; - } - // Field has some kind of type - if (type) { - // If it is an nestec schema - if (value[0]) { - //A nested schema can be just a blank object with no defined paths - if(value[0].tree && value[0].paths){ - cleanTree[field] = getCleanTree(value[0].tree, value[0].paths, ''); - } - // Check for single type arrays (which elasticsearch will treat as the core type i.e. [String] = string) - else if (!paths[field] && prefix) { - if(paths[prefix + field] && paths[prefix + field].caster && paths[prefix + field].caster.instance) { - cleanTree[field] = {type: paths[prefix + field].caster.instance.toLowerCase()}; - } - } else if( paths[field].caster && paths[field].caster.instance ) { - cleanTree[field] = {type: paths[field].caster.instance.toLowerCase()}; - } - else{ - cleanTree[field] = { - type:'object' - }; - } - } else if (value === String || value === Object || value === Date || value === Number || value === Boolean || value === Array){ - cleanTree[field] = {}; - cleanTree[field].type = type; - } else { - cleanTree[field] = value; - cleanTree[field].type = type; - } - - // It has no type for some reason - } else { - // Because it is an geo_point object!! - if (typeof value === 'object' && value.geo_point) { - cleanTree[field] = value.geo_point; - continue; - } - - // If it's a virtual type, don't map it - if (typeof value === 'object' && value.getters && value.setters && value.options) { - continue; - } - - // Because it is some other object!! Or we assumed that it is one. - if (typeof value === 'object') { - cleanTree[field] = getCleanTree(value, paths, prefix + field); - } - } - } - - return cleanTree; + var cleanTree = {}, + type = '', + value = {}; + + if (prefix !== '') { + prefix = prefix + '.'; + } + + for (var field in tree) { + if (prefix === '' && (field === 'id' || field === '_id')) { + continue; + } + + type = getTypeFromPaths(paths, prefix + field); + value = tree[field]; + + if (value.es_indexed === false) { + continue; + } + + // Field has some kind of type + if (type) { + // If it is an nested schema + if (value[0]) { + // A nested array can contain complex objects + if (paths[field] && paths[field].schema && paths[field].schema.tree && paths[field].schema.paths) { + cleanTree[field] = getCleanTree(paths[field].schema.tree, paths[field].schema.paths, ''); + } else if (paths[field] && paths[field].caster && paths[field].caster.instance) { + // Even for simple types the value can be an object if there is other attributes than type + if (typeof value[0] === 'object') { + cleanTree[field] = value[0]; + } else { + cleanTree[field] = {}; + } + + cleanTree[field].type = paths[field].caster.instance.toLowerCase(); + } else if (!paths[field] && prefix) { + if (paths[prefix + field] && paths[prefix + field].caster && paths[prefix + field].caster.instance) { + cleanTree[field] = {type: paths[prefix + field].caster.instance.toLowerCase()}; + } + } else { + cleanTree[field] = { + type: 'object' + }; + } + } else if (value === String || value === Object || value === Date || value === Number || value === Boolean || value === Array) { + cleanTree[field] = {}; + cleanTree[field].type = type; + } else { + cleanTree[field] = value; + cleanTree[field].type = type; + } + + // It has no type for some reason + } else { + // Because it is an geo_* object!! + if (typeof value === 'object') { + var key; + var geoFound = false; + for (key in value) { + if (value.hasOwnProperty(key) && /^geo_/.test(key)) { + cleanTree[field] = value[key]; + geoFound = true; + } + } + + if (geoFound) { + continue; + } + } + + // If it's a virtual type, don't map it + if (typeof value === 'object' && value.getters && value.setters && value.options) { + continue; + } + + // Because it is some other object!! Or we assumed that it is one. + if (typeof value === 'object') { + cleanTree[field] = getCleanTree(value, paths, prefix + field); + } + } + } + + return cleanTree; } - - // // Get type from the mongoose schema // @@ -179,19 +187,19 @@ function getCleanTree(tree, paths, prefix) { // @return the type or false // function getTypeFromPaths(paths, field) { - var type = false; + var type = false; - if (paths[field] && paths[field].options.type === Date) { - return 'date'; - } + if (paths[field] && paths[field].options.type === Date) { + return 'date'; + } - if (paths[field] && paths[field].options.type === Boolean) { - return 'boolean'; - } + if (paths[field] && paths[field].options.type === Boolean) { + return 'boolean'; + } - if (paths[field]) { - type = paths[field].instance ? paths[field].instance.toLowerCase() : 'object'; - } + if (paths[field]) { + type = paths[field].instance ? paths[field].instance.toLowerCase() : 'object'; + } - return type; + return type; } diff --git a/lib/mongoosastic.js b/lib/mongoosastic.js index f0c64751..3d462522 100644 --- a/lib/mongoosastic.js +++ b/lib/mongoosastic.js @@ -1,304 +1,610 @@ -var elastical = require('elastical') - , generator = new(require('./mapping-generator')) - , serialize = require('./serialize') - , events = require('events'); - -module.exports = function elasticSearchPlugin(schema, options){ - var mapping = getMapping(schema) - , indexName = options && options.index - , typeName = options && options.type - , alwaysHydrate = options && options.hydrate - , defaultHydrateOptions = options && options.hydrateOptions - , _mapping = null - , host = options && options.host ? options.host : 'localhost' - , port = options && options.port ? options.port : 9200 - , esClient = new elastical.Client(host, options) - , useRiver = options && options.useRiver; - - if (useRiver) - setUpRiver(schema); - else - setUpMiddlewareHooks(schema); - - /** - * ElasticSearch Client - */ - schema.statics.esClient = esClient; - - /** - * Create the mapping. Takes an optionnal settings parameter and a callback that will be called once - * the mapping is created - - * @param settings String (optional) - * @param callback Function - */ - schema.statics.createMapping = function(settings, cb) { - if (!cb) { - cb = settings; - settings = undefined; - } - setIndexNameIfUnset(this.modelName); - createMappingIfNotPresent(esClient, indexName, typeName, schema, settings, cb); - }; - - /** - * @param indexName String (optional) - * @param typeName String (optional) - * @param callback Function - */ - schema.methods.index = function(index, type, cb){ - if(cb == null && typeof index == 'function'){ - cb = index; - index = null; - }else if (cb == null && typeof type == 'function'){ - cb = type; - type = null - } - var model = this; - setIndexNameIfUnset(model.constructor.modelName); - esClient.index(index || indexName, type || typeName, serialize(model, mapping), {id:model._id.toString()}, cb); - } - - /** - * Unset elastic search index - */ - schema.methods.unIndex = function(){ - var model = this; - setIndexNameIfUnset(model.constructor.modelName); - deleteByMongoId(esClient, model, indexName, typeName, 3); - } - - /** - * Delete all documents from a type/index - * @param callback - callback when truncation is complete - */ - schema.statics.esTruncate = function(cb) { - esClient.delete(indexName, typeName, '', { - query: { - query: { - "match_all": {} - } - } - }, function(err, res) { - cb(err); - }); - } - - /** - * Synchronize an existing collection - * - * @param callback - callback when synchronization is complete - */ - schema.statics.synchronize = function(query){ - var model = this - , em = new events.EventEmitter() - , readyToClose - , closeValues = [] - , counter = 0 - , close = function(){em.emit.apply(em, ['close'].concat(closeValues))} - ; - - setIndexNameIfUnset(model.modelName); - var stream = model.find(query).stream(); - - stream.on('data', function(doc){ - counter++; - doc.save(function(err){ - if (err) { - em.emit('error', err); - return; - } - doc.on('es-indexed', function(err, doc){ - counter--; - if(err){ - em.emit('error', err); - }else{ - em.emit('data', null, doc); - } - if (readyToClose && counter === 0) - close() - }); - }); - }); - stream.on('close', function(a, b){ - readyToClose = true; - closeValues = [a, b]; - if (counter === 0) - close() - }); - stream.on('error', function(err){ - em.emit('error', err); - }); - return em; - }; - /** - * ElasticSearch search function - * - * @param query - query object to perform search with - * @param options - (optional) special search options, such as hydrate - * @param callback - callback called with search results - */ - schema.statics.search = function(query, options, cb){ - var model = this; - setIndexNameIfUnset(model.modelName); - - if(typeof options != 'object'){ - cb = options; - options = {}; - } - query.index = indexName; - esClient.search(query, function(err, results, res){ - if(err){ - cb(err); - }else{ - if (alwaysHydrate || options.hydrate) { - hydrate(results, model, options.hydrateOptions || defaultHydrateOptions || {}, cb); - }else{ - cb(null, res); - } - } - }); - }; - - schema.statics.refresh = function(cb){ - var model = this; - setIndexNameIfUnset(model.modelName); - - esClient.refresh(indexName, cb); - }; - - function setIndexNameIfUnset(model){ - var modelName = model.toLowerCase(); - if(!indexName){ - indexName = modelName + "s"; - } - if(!typeName){ - typeName = modelName; - } - } - - - /** - * Use standard Mongoose Middleware hooks - * to persist to Elasticsearch - */ - function setUpMiddlewareHooks(schema) { - schema.post('remove', function(){ - var model = this; - setIndexNameIfUnset(model.constructor.modelName); - deleteByMongoId(esClient, model, indexName, typeName, 3); - }); - - /** - * Save in elastic search on save. - */ - schema.post('save', function(){ - var model = this; - model.index(function(err, res){ - model.emit('es-indexed', err, res); - }); - }); - } - - /* - * Experimental MongoDB River functionality - * NOTICE: Only tested with: - * MongoDB V2.4.1 - * Elasticsearch V0.20.6 - * elasticsearch-river-mongodb V1.6.5 - * - https://github.com/richardwilly98/elasticsearch-river-mongodb/ - */ - function setUpRiver(schema) { - schema.statics.river = function(cb) { - var model = this; - setIndexNameIfUnset(model.modelName); - if (!this.db.name) throw "ERROR: "+ model.modelName +".river() call before mongoose.connect" - esClient.putRiver( - 'mongodb', - indexName, - { - type: 'mongodb', - mongodb: { - db: this.db.name, - collection: indexName, - gridfs: (useRiver && useRiver.gridfs) ? useRiver.gridfs : false - }, - index: { - name: indexName, - type: typeName - } - }, cb ); - } - } +var elasticsearch = require('elasticsearch') + , generator = new (require('./mapping-generator')) + , serialize = require('./serialize') + , events = require('events') + , mongoose = require('mongoose') + , async = require('async') + , nop = require('nop') + , esClient + +function Mongoosastic(schema, options) { + var mapping = getMapping(schema) + , indexName = options && options.index + , typeName = options && options.type + , alwaysHydrate = options && options.hydrate + , defaultHydrateOptions = options && options.hydrateOptions + , bulk = options.bulk + , bulkBuffer = [] + , bulkTimeout + + this.esClient = this.esClient || new elasticsearch.Client({ + host: options.host || "localhost:9200", + apiVersion:options.apiVersion || '1.0' + }); + esClient = this.esClient; + setUpMiddlewareHooks(schema) + + /** + * ElasticSearch Client + */ + schema.statics.esClient = esClient + + /** + * Create the mapping. Takes an optionnal settings parameter and a callback that will be called once + * the mapping is created + + * @param settings Object (optional) + * @param callback Function + */ + schema.statics.createMapping = function (settings, cb) { + if (arguments.length < 2) { + cb = arguments[0] || nop + settings = undefined + } + + setIndexNameIfUnset(this.modelName) + + createMappingIfNotPresent({ + client: esClient, + indexName: indexName, + typeName: typeName, + schema: schema, + settings: settings + }, cb) + } + + /** + * @param options Object (optional) + * @param callback Function + */ + schema.methods.index = function (options, cb) { + var _this = this; + if (arguments.length < 2) { + cb = arguments[0] || nop + options = {} + } + + //自动匹配index与type + setIndexNameIfUnset(this.constructor.modelName) + var index = options.index || indexName + , type = options.type || typeName + ,serialModel = serialize(this, mapping); + createMappingIfNotPresent({ + client: esClient, + indexName: index, + typeName: type, + schema: schema + },function(err){ + if(err) throw err; + if (bulk) { + /** + * To serialize in bulk it needs the _id + */ + serialModel._id = _this._id; + bulkIndex({ + index: index, + type: type, + model: serialModel + }) + setImmediate(cb) + } else { + esClient.index({ + index: index, + type: type, + id: _this._id.toString(), + body: serialModel + }, cb) + } + + }); + } + + /** + * Unset elastic search index + * @param options - (optional) options for unIndex + * @param callback - callback when unIndex is complete + */ + schema.methods.unIndex = function (options, cb) { + if (arguments.length < 2) { + cb = arguments[0] || nop + options = {} + } + + setIndexNameIfUnset(this.constructor.modelName) + + options.index = options.index || indexName + options.type = options.type || typeName + options.model = this + options.client = esClient + options.tries = 3 + + if (bulk) + bulkDelete(options, cb) + else + deleteByMongoId(options, cb) + } + + /** + * Delete all documents from a type/index + * @param options - (optional) specify index/type + * @param callback - callback when truncation is complete + */ + schema.statics.esTruncate = function (options, cb) { + if (arguments.length < 2) { + cb = arguments[0] || nop + options = {} + } + + var index = options.index || indexName + , type = options.type || typeName + + esClient.deleteByQuery({ + index: index, + type: type, + body: { + query: { + match_all: {} + } + } + }, cb) + } + + /** + * Synchronize an existing collection + * + * @param query - query for documents you want to synchronize + */ + schema.statics.synchronize = function (query) { + var em = new events.EventEmitter() + , closeValues = [] + , counter = 0 + , close = function () { + em.emit.apply(em, ['close'].concat(closeValues)) + } + + //Set indexing to be bulk when synchronizing to make synchronizing faster + bulk = bulk || { + delay: 1000, + size: 1000 + } + + query = query || {} + + setIndexNameIfUnset(this.modelName) + + var stream = this.find(query).stream() + + stream.on('data', function (doc) { + counter++ + doc.save(function (err) { + if (err) + return em.emit('error', err) + + doc.on('es-indexed', function (err, doc) { + counter-- + if (err) { + em.emit('error', err) + } else { + em.emit('data', null, doc) + } + }) + }) + }) + + stream.on('close', function (a, b) { + closeValues = [a, b] + var closeInterval = setInterval(function () { + if (counter === 0 && bulkBuffer.length === 0) { + clearInterval(closeInterval) + close() + } + }, 1000) + }) + + stream.on('error', function (err) { + em.emit('error', err) + }) + + return em + } + /** + * ElasticSearch search function + * + * @param query - query object to perform search with + * @param options - (optional) special search options, such as hydrate + * @param callback - callback called with search results + */ + schema.statics.search = function (query, options, cb) { + if (arguments.length === 2) { + cb = arguments[1] + options = {} + } + + if (query === null) + query = undefined + + setIndexNameIfUnset(this.modelName) + + var model = this + , esQuery = { + body: {query: query}, + index: options.index || indexName, + type: options.type || typeName + } + + + Object.keys(options).forEach(function (opt) { + if (!opt.match(/hydrate/) && options.hasOwnProperty(opt)) + esQuery[opt] = options[opt] + }) + + esClient.search(esQuery, function (err, res) { + if (err) { + cb(err) + } else { + if (alwaysHydrate || options.hydrate) { + hydrate(res,options.hydrateOptions || defaultHydrateOptions || {}, cb) + } else { + cb(null, res) + } + } + }) + } + + function bulkDelete(options, cb) { + bulkAdd({ + delete: { + _index: options.index || indexName, + _type: options.type || typeName, + _id: options.model._id.toString() + } + }) + cb() + } + + function bulkIndex(options) { + bulkAdd({ + index: { + _index: options.index || indexName, + _type: options.type || typeName, + _id: options.model._id.toString() + } + }) + bulkAdd(options.model) + } + + function clearBulkTimeout() { + clearTimeout(bulkTimeout) + bulkTimeout = undefined + } + + function bulkAdd(instruction) { + bulkBuffer.push(instruction) + + //Return because we need the doc being indexed + //Before we start inserting + if (instruction.index && instruction.index._index) + return + + if (bulkBuffer.length >= (bulk.size || 1000)) { + schema.statics.flush() + clearBulkTimeout() + } else if (bulkTimeout === undefined) { + bulkTimeout = setTimeout(function () { + schema.statics.flush() + clearBulkTimeout() + }, bulk.delay || 1000) + } + } + + schema.statics.flush = function (cb) { + cb = cb || function (err) { + if (err) console.log(err) + } + + esClient.bulk({ + body: bulkBuffer + }, function (err) { + cb(err) + }) + bulkBuffer = [] + } + + schema.statics.refresh = function (options, cb) { + if (arguments.length < 2) { + cb = arguments[0] || nop + options = {} + } + + setIndexNameIfUnset(this.modelName) + esClient.indices.refresh({ + index: options.index || indexName + }, cb) + } + + function setIndexNameIfUnset(model) { + var modelName = model.toLowerCase() + if (!indexName) { + indexName = modelName + "s" + } + if (!typeName) { + typeName = modelName + } + } + + + /** + * Use standard Mongoose Middleware hooks + * to persist to Elasticsearch + */ + function setUpMiddlewareHooks(schema) { + schema.post('remove', function () { + setIndexNameIfUnset(this.constructor.modelName) + + var options = { + index: indexName, + type: typeName, + tries: 3, + model: this, + client: esClient + } + + if (bulk) { + bulkDelete(options, nop) + } else { + deleteByMongoId(options, nop) + } + }) + + /** + * Save in elastic search on save. + */ + schema.post('save', function () { + var model = this + + model.index(function (err, res) { + model.emit('es-indexed', err, res) + }) + }) + } + +} + + +module.exports = { + mongoose:null, + connect: function (options) { + var host = options && options.host || 'localhost', + port = options && options.port || 9200, + protocol = options && options.protocol || 'http', + auth = options && options.auth ? options.auth : null; + + this.esClient = this.esClient || new elasticsearch.Client({ + host: { + host: host, + port: port, + protocol: protocol, + auth: auth + } + }); + }, + suggest:function(query, options, cb){ + if (arguments.length === 2) { + cb = arguments[1] + options = {} + } + mongoose = this.mongoose || mongoose; + if (query === null) + query = undefined + + var esQuery = { + body:query, + index: options.index || "", + type: options.type || "" + }; + + esClient.indices.exists({index: options.index},function(err,exists){ + if(!err){ + if(exists){ + esClient.suggest(esQuery, function (err, res) { + if (err) { + cb(err) + } else { + cb(null, res) + } + }) + } else { + cb(null, {}); + } + } + }); + }, + /** + * 全局性的搜索，可以搜索多索引，多类型，同时也能hydrate + */ + search:function(query, options, cb){ + if (arguments.length === 2) { + cb = arguments[1] + options = {} + } + mongoose = this.mongoose || mongoose; + if (query === null) + query = undefined + + var esQuery = { + body:query, + index: options.index || "", + type: options.type || "" + }; + + Object.keys(options).forEach(function (opt) { + if (!opt.match(/hydrate/) && options.hasOwnProperty(opt)) + esQuery[opt] = options[opt] + }) + + esClient.indices.exists({index: options.index},function(err,exists){ + if(!err){ + if(exists){ + esClient.search(esQuery, function (err, res) { + if (err) { + cb(err) + } else { + if (options.hydrate) { + hydrate(res,options || {}, cb) + } else { + cb(null, res) + } + } + }) + } else { + cb(null, {}); + } + } + }); + + }, + plugin: function(options){ + var _this = this; + options = options || {}; + options.host = options && options.host ? options.host : 'localhost:9200' + options.apiVersion = options && options.apiVersion ? options.apiVersion : '1.0' + return function(schema,_options){ + _options = _options || {}; + _this.options = extend(options,_options); + return Mongoosastic(schema,_this.options); + } + } }; +function extend(target) { + var src + for (var i = 1, l = arguments.length; i < l; i++) { + src = arguments[i] + for (var k in src) target[k] = src[k] + } + return target +} + +function createMappingIfNotPresent(options, cb) { + var client = options.client + , indexName = options.indexName + , typeName = options.typeName + , schema = options.schema + , settings = options.settings + + generator.generateMapping(schema, function (err, mapping) { + var completeMapping = {} + completeMapping[typeName] = mapping + client.indices.exists({index: indexName}, function (err, exists) { + if (err) + return cb(err) + if (exists) { + client.indices.putMapping({ + index: indexName, + type: typeName, + body: completeMapping + }, cb) + } else { + client.indices.create({index: indexName, body: settings}, function (err) { + if (err) + return cb(err) + client.indices.putMapping({ + index: indexName, + type: typeName, + body: completeMapping + }, cb) + }) + } + }) + }) +} -function createMappingIfNotPresent(client, indexName, typeName, schema, settings, cb) { - generator.generateMapping(schema, function(err, mapping) { - var completeMapping = {}; - completeMapping[typeName] = mapping; - client.indexExists(indexName, function(err, exists) { - if (exists) { - client.putMapping(indexName, typeName, completeMapping, cb); - } else { - client.createIndex(indexName, { - settings: settings, - mappings: completeMapping - }, cb); - } - }); - }); +function hydrate(res,options, cb) { + var results = res.hits + , resultsMap = {} + , ids = {} + , querys = {} + , hits = [] + , model + results.hits.forEach(function(a,i){ + var modelName = getModelName(a); + if(modelName) { + resultsMap[modelName] = resultsMap[modelName] || {}; + ids[modelName] = ids[modelName] || []; + resultsMap[modelName][a._id] = i;//记录排序索引 + ids[modelName].push(a._id); + } + }); + async.eachSeries(Object.keys(resultsMap),function(modelName,callback){ + model = mongoose.model(modelName); + querys[modelName] = model.find({_id:{$in:ids[modelName]}}); + Object.keys(options.hydrateOptions).forEach(function (option) { + querys[modelName][option](options.hydrateOptions[option]) + }) + querys[modelName].exec(function(err, docs){ + if (err) { + return cb(err) + } else { + docs.forEach(function (doc) { + var i = resultsMap[modelName][doc._id] + hits[i] = Object.create(doc); + }); + callback(); + } + }) + },function(){ + results.hits = hits + res.hits = results + cb(null, res) + }); } -function hydrate(results, model, options, cb){ - var resultsMap = {} - var ids = results.hits.map(function(a, i){ - resultsMap[a._id] = i - return a._id; - }); - var query = model.find({_id:{$in:ids}}); - - // Build Mongoose query based on hydrate options - // Example: {lean: true, sort: '-name', select: 'address name'} - Object.keys(options).forEach(function(option){ - query[option](options[option]); - }); - - query.exec(function(err, docs){ - if(err){ - return cb(err); - }else{ - var hits = []; - - docs.forEach(function(doc) { - var i = resultsMap[doc._id] - hits[i] = doc - }) - results.hits = hits; - cb(null, results); - } - }); +function getModelName(es_item){ + if(!es_item || !es_item._type) return; + var names = mongoose.modelNames(), + res=""; + names.forEach(function(name){ + if(es_item._type === name.toLowerCase()){ + res = name; + return false; + } + }); + return res; } -function getMapping(schema){ - var retMapping = {}; - generator.generateMapping(schema, function(err, mapping){ - retMapping = mapping; - }); - return retMapping; + +function getMapping(schema) { + var retMapping = {} + generator.generateMapping(schema, function (err, mapping) { + retMapping = mapping + }) + return retMapping } -function deleteByMongoId(client, model,indexName, typeName, tries){ - client.delete(indexName, typeName, model._id.toString(), function(err, res){ - if(err && err.message.indexOf('404') > -1){ - setTimeout(function(){ - if(tries <= 0){ - // future issue.. what do we do!? - }else{ - deleteByMongoId(client, model, indexName, typeName, --tries); - } - }, 500); - }else{ - model.emit('es-removed', err, res); - } - }); + +function deleteByMongoId(options, cb) { + var index = options.index + , type = options.type + , client = options.client + , model = options.model + , tries = options.tries + + client.delete({ + index: index, + type: type, + id: model._id.toString() + }, function (err, res) { + if (err && err.message.indexOf('404') > -1) { + setTimeout(function () { + if (tries <= 0) { + return cb(err) + } else { + options.tries = --tries + deleteByMongoId(options, cb) + } + }, 500) + } else { + model.emit('es-removed', err, res) + cb(err) + } + }) } diff --git a/lib/serialize.js b/lib/serialize.js index bef81e66..384d7f3c 100644 --- a/lib/serialize.js +++ b/lib/serialize.js @@ -1,33 +1,50 @@ module.exports = serialize; +function _serializeObject(object, mapping) { + var serialized = {}; + for (var field in mapping.properties) { + var val = serialize.call(object, object[field], mapping.properties[field]); + if (val !== undefined) { + serialized[field] = val; + } + } + + return serialized; +} + function serialize(model, mapping) { + var name; + + if (mapping.properties && model) { + + if (Array.isArray(model)) { + return model.map(function (object) { + return _serializeObject(object, mapping); + }); + } + + return _serializeObject(model, mapping); + + } + + if (mapping.cast && typeof mapping.cast !== 'function') { + throw new Error('es_cast must be a function'); + } + + model = mapping.cast ? mapping.cast.call(this, model) : model; + if (typeof model === 'object' && model !== null) { + name = model.constructor.name; + if (name === 'ObjectID') { + return model.toString(); + } + + if (name === 'Date') { + return new Date(model).toJSON(); + } + + } + + return model; + +} - if (mapping.properties) { - var serializedForm = {}; - - for (var field in mapping.properties) { - var val = serialize(model[field], mapping.properties[field]); - if (val !== undefined) { - serializedForm[field] = val; - } - } - - return serializedForm; - - } else { - if (mapping.cast && typeof(mapping.cast) !== 'function') - throw new Error('es_cast must be a function'); - model = mapping.cast ? mapping.cast(model) : model; - if (typeof model === 'object' && model !== null) { - var name = model.constructor.name; - if (name === 'ObjectID') { - return model.toString(); - } else if (name === 'Date') { - return new Date(model).toJSON(); - } - return model; - } else { - return model; - } - } -} \ No newline at end of file diff --git a/package.json b/package.json index c90295c1..3dc3ce0b 100644 --- a/package.json +++ b/package.json @@ -1,30 +1,69 @@ { - "author": "James R. Carr (http://blog.james-carr.org)", + "author": { + "name": "James R. Carr", + "email": "james.r.carr@gmail.com", + "url": "http://blog.james-carr.org" + }, "name": "mongoosastic", "description": "A mongoose plugin that indexes models into elastic search", - "version": "0.6.0", - "tags":["mongodb", "elastic search", "mongoose", "full text search"], + "version": "2.0.6", + "tags": [ + "mongodb", + "elastic search", + "mongoose", + "full text search" + ], "repository": { "type": "git", - "url": "git://github.com/jamescarr/mongoosastic" + "url": "https://github.com/janryWang/mongoosastic" }, - "main":"lib/mongoosastic.js", + "main": "lib/mongoosastic.js", "dependencies": { - "elastical":"0.0.12" + "elasticsearch": "^2.4.3", + "nop": "^1.0.0" }, "peerDependencies": { - "mongoose":"3.8.x" + "mongoose": "^3.8.22" }, "devDependencies": { - "mocha":"*" - , "should":"*" - , "async":"*" - , "mongoose":"3.8.x" + "mocha": "*", + "should": "*", + "async": "*", + "mongoose": "3.8.x" }, "engines": { "node": ">= 0.8.0" }, - "scripts":{ - "test":"mocha -R spec -t 20000 -b" - } + "scripts": { + "test": "mocha -R spec -t 20000 -b" + }, + "gitHead": "a1c25990cca9717497c0e706d7ac64b5ed204819", + "bugs": { + "url": "https://github.com/mongoosastic/mongoosastic/issues" + }, + "homepage": "https://github.com/mongoosastic/mongoosastic", + "_id": "mongoosastic@2.0.6", + "_shasum": "8c86d2e396cf110abbcc9ef3e041818a1b4f9788", + "_from": "mongoosastic@", + "_npmVersion": "2.0.0", + "_npmUser": { + "name": "taterbase", + "email": "taterbase@gmail.com" + }, + "maintainers": [ + { + "name": "jamescarr", + "email": "james.r.carr@gmail.com" + }, + { + "name": "taterbase", + "email": "shankga@gmail.com" + } + ], + "dist": { + "shasum": "8c86d2e396cf110abbcc9ef3e041818a1b4f9788", + "tarball": "http://registry.npmjs.org/mongoosastic/-/mongoosastic-2.0.6.tgz" + }, + "directories": {}, + "_resolved": "https://registry.npmjs.org/mongoosastic/-/mongoosastic-2.0.6.tgz" } diff --git a/test/alternative-index-method-test.js b/test/alternative-index-method-test.js old mode 100644 new mode 100755 index 1d43eb01..72534caf --- a/test/alternative-index-method-test.js +++ b/test/alternative-index-method-test.js @@ -1,22 +1,21 @@ var mongoose = require('mongoose') - , elastical = require('elastical') , should = require('should') , config = require('./config') , Schema = mongoose.Schema , ObjectId = Schema.ObjectId - , esClient = new(require('elastical').Client) , mongoosastic = require('../lib/mongoosastic') , Tweet = require('./models/tweet'); - describe('Index Method', function(){ before(function(done){ mongoose.connect(config.mongoUrl, function(){ config.deleteIndexIfExists(['tweets', 'public_tweets'], function(){ - config.createModelAndEnsureIndex(Tweet, { - user: 'jamescarr' - , message: "I know kung-fu!" - , post_date: new Date() - }, done); + Tweet.remove(function() { + config.createModelAndEnsureIndex(Tweet, { + user: 'jamescarr' + , message: "I know kung-fu!" + , post_date: new Date() + }, done); + }) }); }); }); @@ -27,12 +26,13 @@ describe('Index Method', function(){ done(); }); }); + it('should be able to index it directly without saving', function(done){ Tweet.findOne({message:'I know kung-fu!'}, function(err, doc){ doc.message = 'I know nodejitsu!'; doc.index(function(){ setTimeout(function(){ - Tweet.search({query:'know'}, function(err, res){ + Tweet.search({query_string: {query: 'know'}}, function(err, res){ res.hits.hits[0]._source.message.should.eql('I know nodejitsu!'); done(); }); @@ -43,9 +43,9 @@ describe('Index Method', function(){ it('should be able to index to alternative index', function(done){ Tweet.findOne({message:'I know kung-fu!'}, function(err, doc){ doc.message = 'I know taebo!'; - doc.index('public_tweets', function(){ + doc.index({index: 'public_tweets'}, function(){ setTimeout(function(){ - esClient.search({index: 'public_tweets', query:'know'}, function(err, results, res){ + Tweet.search({query_string: {query: 'know'}}, {index: 'public_tweets'}, function(err, res){ res.hits.hits[0]._source.message.should.eql('I know taebo!'); done(); }); @@ -56,9 +56,9 @@ describe('Index Method', function(){ it('should be able to index to alternative index and type', function(done){ Tweet.findOne({message:'I know kung-fu!'}, function(err, doc){ doc.message = 'I know taebo!'; - doc.index('public_tweets', 'utterings', function(){ + doc.index({index: 'public_tweets', type: 'utterings'}, function(){ setTimeout(function(){ - esClient.search({index: 'public_tweets', type: 'utterings', query:'know'}, function(err, results, res){ + Tweet.search({query_string: {query: 'know'}}, {index: 'public_tweets', type: 'utterings'}, function(err, res){ res.hits.hits[0]._source.message.should.eql('I know taebo!'); done(); }); diff --git a/test/boost-field-test.js b/test/boost-field-test.js old mode 100644 new mode 100755 index e9b94108..70b84b86 --- a/test/boost-field-test.js +++ b/test/boost-field-test.js @@ -1,6 +1,5 @@ var mongoose = require('mongoose') - , elastical = require('elastical') - , esClient = new(require('elastical').Client) + , esClient = new(require('elasticsearch').Client) , should = require('should') , config = require('./config') , Schema = mongoose.Schema @@ -15,7 +14,7 @@ var TweetSchema = new Schema({ , title: {type:String, es_boost:2.0} }); -TweetSchema.plugin(mongoosastic); +TweetSchema.plugin(mongoosastic.plugin()); var BlogPost = mongoose.model('BlogPost', TweetSchema); describe('Add Boost Option Per Field', function(){ @@ -29,8 +28,16 @@ describe('Add Boost Option Per Field', function(){ it('should create a mapping with boost field added', function(done){ BlogPost.createMapping(function(err, mapping){ - esClient.getMapping('blogposts', 'blogpost', function(err, mapping){ - var props = mapping.blogposts.mappings.blogpost.properties; + esClient.indices.getMapping({ + index: 'blogposts', + type: 'blogpost' + }, function(err, mapping){ + + /* elasticsearch 1.0 & 0.9 support */ + var props = mapping.blogpost != undefined ? + mapping.blogpost.properties : /* ES 0.9.11 */ + mapping.blogposts.mappings.blogpost.properties; /* ES 1.0.0 */ + props.title.type.should.eql('string'); props.title.boost.should.eql(2.0); done(); diff --git a/test/bulk-test.js b/test/bulk-test.js new file mode 100755 index 00000000..b581ba91 --- /dev/null +++ b/test/bulk-test.js @@ -0,0 +1,69 @@ +var mongoose = require('mongoose'), + should = require('should'), + config = require('./config'), + Schema = mongoose.Schema, + ObjectId = Schema.ObjectId, + async = require('async'), + mongoosastic = require('../lib/mongoosastic'); + +var BookSchema = new Schema({ + title: String +}); +BookSchema.plugin(mongoosastic.plugin(), { + bulk: { + size: 10, + delay: 100 + } +}); + +var Book = mongoose.model('Book2', BookSchema); + +describe('Bulk mode', function() { + var books = null; + + before(function(done) { + config.deleteIndexIfExists(['book2s'], function() { + mongoose.connect(config.mongoUrl, function() { + var client = mongoose.connections[0].db; + client.collection('book2s', function(err, _books) { + books = _books; + Book.remove(done); + }); + }); + }); + }); + before(function(done) { + async.forEach(bookTitles(), function(title, cb) { + new Book({ + title: title + }).save(cb); + }, done) + }); + before(function(done) { + Book.findOne({ + title: 'American Gods' + }, function(err, book) { + book.remove(done) + }); + }); + it('should index all objects and support deletions too', function(done) { + setTimeout(function() { + Book.search({match_all: {}}, function(err, results) { + results.should.have.property('hits').with.property('total', 52); + done(); + }); + }, 3000) + }); +}); + +function bookTitles() { + var books = [ + 'American Gods', + 'Gods of the Old World', + 'American Gothic' + ]; + for (var i = 0; i < 50; i++) { + books.push('ABABABA' + i); + } + return books; +} diff --git a/test/config.js b/test/config.js old mode 100644 new mode 100755 index 2fad571a..7e3f3c5d --- a/test/config.js +++ b/test/config.js @@ -1,22 +1,26 @@ -var esClient = new(require('elastical').Client) - , async = require('async'); +var esClient = new(require('elasticsearch').Client) + , async = require('async'); const INDEXING_TIMEOUT = 1100; module.exports = { - mongoUrl: 'mongodb://localhost/es-test' + mongoUrl: 'mongodb://localhost/es-test' , indexingTimeout: INDEXING_TIMEOUT , deleteIndexIfExists: function(indexes, done){ - async.forEach(indexes, function(index, cb){ - esClient.indexExists(index, function(err, exists){ - if(exists){ - esClient.deleteIndex(index, cb); - }else{ - cb(); - } - }); - }, done); - } + async.forEach(indexes, function(index, cb){ + esClient.indices.exists({ + index: index + }, function(err, exists){ + if(exists){ + esClient.indices.delete({ + index: index + }, cb); + }else{ + cb(); + } + }); + }, done); + } , createModelAndEnsureIndex: createModelAndEnsureIndex }; @@ -27,4 +31,4 @@ function createModelAndEnsureIndex(model, obj, cb){ setTimeout(cb, INDEXING_TIMEOUT); }); }); -} +} \ No newline at end of file diff --git a/test/geo-test.js b/test/geo-test.js new file mode 100755 index 00000000..31a888df --- /dev/null +++ b/test/geo-test.js @@ -0,0 +1,190 @@ +var mongoose = require('mongoose') + , esClient = new(require('elasticsearch').Client) + , should = require('should') + , config = require('./config') + , Schema = mongoose.Schema + , ObjectId = Schema.ObjectId + , mongoosastic = require('../lib/mongoosastic'); + + +var GeoSchema; + + +var GeoModel; + +describe('GeoTest', function(){ + before(function(done){ + mongoose.connect(config.mongoUrl, function(){ + config.deleteIndexIfExists(['geodocs'], function(){ + + GeoSchema = new Schema({ + myId: Number, + frame: { + coordinates : [], + type: {type: String}, + geo_shape: { + type:String, + es_type: "geo_shape", + es_tree: "quadtree", + es_precision: "1km" + } + } + }); + + GeoSchema.plugin(mongoosastic.plugin()); + GeoModel = mongoose.model('geodoc', GeoSchema); + + GeoModel.createMapping(function(err, mapping){ + GeoModel.remove(function(){ + + esClient.indices.getMapping({ + index: 'geodocs', + type: 'geodoc' + }, function(err, mapping){ + (mapping.geodoc != undefined ? + mapping.geodoc: /* ES 0.9.11 */ + mapping.geodocs.mappings.geodoc /* ES 1.0.0 */ + ).properties.frame.type.should.eql('geo_shape'); + done(); + }); + }); + }); + + }); + }); + }); + + it('should be able to create and store geo coordinates', function(done){ + + var geo = new GeoModel({ + myId : 1, + frame:{ + type:'envelope', + coordinates: [[1,4],[3,2]] + } + }); + + geo2 = new GeoModel({ + myId : 2, + frame:{ + type:'envelope', + coordinates: [[2,3],[4,0]] + } + }); + + + var saveAndWait = function (doc,cb) { + doc.save(function(err) { + if (err) cb(err); + else doc.on('es-indexed', cb ); + }); + }; + + saveAndWait(geo,function(err){ + if (err) throw err; + saveAndWait(geo2,function(err){ + if (err) throw err; + // Mongodb request + GeoModel.find({},function(err, res){ + if (err) throw err; + res.length.should.eql(2); + res[0].frame.type.should.eql('envelope'); + res[0].frame.coordinates[0].should.eql([1,4]); + res[0].frame.coordinates[1].should.eql([3,2]); + done(); + })})})}) + + it('should be able to find geo coordinates in the indexes', function(done){ + setTimeout(function(){ + // ES request + GeoModel.search({ + match_all: {} + }, {sort: "myId:asc"}, function(err, res){ + if (err) throw err; + res.hits.total.should.eql(2); + res.hits.hits[0]._source.frame.type.should.eql('envelope'); + res.hits.hits[0]._source.frame.coordinates.should.eql([[1,4],[3,2]]); + done(); + }); + }, 1100); + }); + + it('should be able to resync geo coordinates from the database', function(done){ + config.deleteIndexIfExists(['geodocs'], function(){ + GeoModel.createMapping(function(err, mapping){ + var stream = GeoModel.synchronize() + , count = 0; + + stream.on('data', function(err, doc){ + count++; + }); + + stream.on('close', function(){ + count.should.eql(2); + + setTimeout(function(){ + GeoModel.search({ + match_all: {} + }, {sort: "myId:asc"}, function(err, res){ + if (err) throw err; + res.hits.total.should.eql(2); + res.hits.hits[0]._source.frame.type.should.eql('envelope'); + res.hits.hits[0]._source.frame.coordinates.should.eql([[1,4],[3,2]]); + done(); + }); + }, 1000); + }); + }); + }); + }); + + + + it('should be able to search points inside frames', function(done){ + var geoQuery = { + filtered: { + "query": {"match_all": {}}, + "filter": { + "geo_shape": { + "frame": { + "shape": { + "type": "point", + "coordinates": [3,1] + } + } + } + } + } + } + + setTimeout(function(){ + GeoModel.search(geoQuery,function(err, res){ + if (err) throw err; + res.hits.total.should.eql(1); + res.hits.hits[0]._source.myId.should.eql(2); + geoQuery.filtered.filter.geo_shape.frame.shape.coordinates = [1.5,2.5]; + GeoModel.search(geoQuery,function(err, res){ + if (err) throw err; + res.hits.total.should.eql(1); + res.hits.hits[0]._source.myId.should.eql(1); + + geoQuery.filtered.filter.geo_shape.frame.shape.coordinates = [3,2]; + GeoModel.search(geoQuery,function(err, res){ + if (err) throw err; + res.hits.total.should.eql(2); + + geoQuery.filtered.filter.geo_shape.frame.shape.coordinates = [0,3]; + GeoModel.search(geoQuery,function(err, res){ + if (err) throw err; + res.hits.total.should.eql(0); + done(); + }); + }); + }); + + }); + }, 1000); + }); + + +}); diff --git a/test/index-test.js b/test/index-test.js old mode 100644 new mode 100755 index 711bb5f3..55c71dc2 --- a/test/index-test.js +++ b/test/index-test.js @@ -1,10 +1,9 @@ var mongoose = require('mongoose') - , elastical = require('elastical') , should = require('should') , config = require('./config') , Schema = mongoose.Schema , ObjectId = Schema.ObjectId - , esClient = new(require('elastical').Client) + , esClient = new(require('elasticsearch').Client) , mongoosastic = require('../lib/mongoosastic') , Tweet = require('./models/tweet'); @@ -16,7 +15,7 @@ var TalkSchema = new Schema({ , abstract: {type:String, es_indexed:true} , bio: String }); -TalkSchema.plugin(mongoosastic) +TalkSchema.plugin(mongoosastic.plugin()) var Talk = mongoose.model("Talk", TalkSchema); @@ -29,7 +28,7 @@ var PersonSchema = new Schema({ , died: {type: Number, es_indexed:true} } }); -PersonSchema.plugin(mongoosastic, { +PersonSchema.plugin(mongoosastic.plugin(), { index:'people' , type: 'dude' , hydrate: true @@ -99,27 +98,41 @@ describe('indexing', function(){ }); it("should use the model's id as ES id", function(done){ Tweet.findOne({message:"I like Riak better"}, function(err, doc){ - esClient.get('tweets', doc._id.toString(), function(err, res){ - res.message.should.eql(doc.message); + esClient.get({ + index: 'tweets', + type: 'tweet', + id: doc._id.toString() + }, function(err, res){ + res._source.message.should.eql(doc.message); done() }); }); }); it('should be able to execute a simple query', function(done){ - Tweet.search({query:'Riak'}, function(err, results) { + Tweet.search({ + query_string: { + query: 'Riak' + } + }, function(err, results) { results.hits.total.should.eql(1) results.hits.hits[0]._source.message.should.eql('I like Riak better') done(); }); }); + it('should be able to execute a simple query', function(done){ - Tweet.search({query:'jamescarr'}, function(err, results) { + Tweet.search({ + query_string: { + query: 'jamescarr' + } + }, function(err, results) { results.hits.total.should.eql(1) results.hits.hits[0]._source.message.should.eql('I like Riak better') done() }); }); + it('should report errors', function(done){ Tweet.search({queriez:'jamescarr'}, function(err, results) { err.message.should.match(/SearchPhaseExecutionException/); @@ -140,7 +153,11 @@ describe('indexing', function(){ it('should remove from index when model is removed', function(done){ tweet.remove(function(){ setTimeout(function(){ - Tweet.search({query:'shouldnt'}, function(err, res){ + Tweet.search({ + query_string: { + query: 'shouldnt' + } + }, function(err, res){ res.hits.total.should.eql(0); done(); }); @@ -150,7 +167,11 @@ describe('indexing', function(){ it('should remove only index', function(done){ tweet.on('es-removed', function(err, res){ setTimeout(function(){ - Tweet.search({query:'shouldnt'}, function(err, res){ + Tweet.search({ + query_string: { + query: 'shouldnt' + } + }, function(err, res){ res.hits.total.should.eql(0); done(); }); @@ -197,14 +218,14 @@ describe('indexing', function(){ }); it('should only find models of type Tweet', function(done){ - Tweet.search({query:'Dude'}, function(err, res){ + Tweet.search({query_string: {query: 'Dude'}}, function(err, res){ res.hits.total.should.eql(1); res.hits.hits[0]._source.user.should.eql('Dude'); done(); }); }); it('should only find models of type Talk', function(done){ - Talk.search({query:'Dude'}, function(err, res){ + Talk.search({query_string: {query: 'Dude'}}, function(err, res){ res.hits.total.should.eql(1); res.hits.hits[0]._source.title.should.eql('Dude'); done(); @@ -222,11 +243,11 @@ describe('indexing', function(){ }); it('when gathering search results while respecting default hydrate options', function(done){ - Person.search({query:'James'}, function(err, res) { - res.hits[0].address.should.eql('Exampleville, MO'); - res.hits[0].name.should.eql('James Carr'); - res.hits[0].should.not.have.property('phone'); - res.hits[0].should.not.be.an.instanceof(Person); + Person.search({query_string: {query: 'James'}}, function(err, res) { + res.hits.hits[0].address.should.eql('Exampleville, MO'); + res.hits.hits[0].name.should.eql('James Carr'); + res.hits.hits[0].should.not.have.property('phone'); + res.hits.hits[0].should.not.be.an.instanceof(Person); done(); }); }); @@ -243,7 +264,7 @@ describe('indexing', function(){ }); it('should only return indexed fields', function(done){ - Talk.search({query:'cool'}, function(err, res) { + Talk.search({query_string: {query: 'cool'}}, function(err, res) { res.hits.total.should.eql(1); var talk = res.hits.hits[0]._source; @@ -257,10 +278,10 @@ describe('indexing', function(){ }); it('should hydrate returned documents if desired', function(done){ - Talk.search({query:'cool'}, {hydrate:true}, function(err, res) { - res.total.should.eql(1) + Talk.search({query_string: {query: 'cool'}}, {hydrate:true}, function(err, res) { + res.hits.total.should.eql(1) - var talk = res.hits[0] + var talk = res.hits.hits[0] talk.should.have.property('title') talk.should.have.property('year'); talk.should.have.property('abstract') @@ -282,25 +303,25 @@ describe('indexing', function(){ }); it('should only return indexed fields and have indexed sub-objects', function(done){ - Person.search({query:'Bob'}, function(err, res) { - res.hits[0].address.should.eql('Exampleville, MO'); - res.hits[0].name.should.eql('Bob Carr'); - res.hits[0].should.have.property('life'); - res.hits[0].life.born.should.eql(1950); - res.hits[0].life.should.not.have.property('died'); - res.hits[0].life.should.not.have.property('other'); - res.hits[0].should.not.have.property('phone'); - res.hits[0].should.not.be.an.instanceof(Person); + Person.search({query_string: {query: 'Bob'}}, function(err, res) { + res.hits.hits[0].address.should.eql('Exampleville, MO'); + res.hits.hits[0].name.should.eql('Bob Carr'); + res.hits.hits[0].should.have.property('life'); + res.hits.hits[0].life.born.should.eql(1950); + res.hits.hits[0].life.should.not.have.property('died'); + res.hits.hits[0].life.should.not.have.property('other'); + res.hits.hits[0].should.not.have.property('phone'); + res.hits.hits[0].should.not.be.an.instanceof(Person); done(); }); }); }); it('should allow extra query options when hydrating', function(done){ - Talk.search({query:'cool'}, {hydrate:true, hydrateOptions: {lean: true}}, function(err, res) { - res.total.should.eql(1) + Talk.search({query_string: {query: 'cool'}}, {hydrate:true, hydrateOptions: {lean: true}}, function(err, res) { + res.hits.total.should.eql(1) - var talk = res.hits[0] + var talk = res.hits.hits[0] talk.should.have.property('title') talk.should.have.property('year'); talk.should.have.property('abstract') @@ -316,13 +337,18 @@ describe('indexing', function(){ describe('Existing Index', function(){ before(function(done){ config.deleteIndexIfExists(['ms_sample'], function(){ - esClient.createIndex('ms_sample', {mappings:{ - bum:{ - properties: { - name: {type:'string'} + esClient.indices.create({ + index: 'ms_sample', + body: { + mappings:{ + bum:{ + properties: { + name: {type:'string'} + } + } } } - }}, done); + }, done); }); }); @@ -330,13 +356,13 @@ describe('indexing', function(){ var BumSchema = new Schema({ name: String }); - BumSchema.plugin(mongoosastic, { + BumSchema.plugin(mongoosastic.plugin(), { index: 'ms_sample' , type: 'bum' }); var Bum = mongoose.model('bum', BumSchema); config.createModelAndEnsureIndex(Bum, {name:'Roger Wilson'}, function(){ - Bum.search({query:'Wilson'}, function(err, results){ + Bum.search({query_string: {query: 'Wilson'}}, function(err, results){ results.hits.total.should.eql(1); done(); }); diff --git a/test/mapping-generator-test.js b/test/mapping-generator-test.js old mode 100644 new mode 100755 index 0d51207c..61d07fc4 --- a/test/mapping-generator-test.js +++ b/test/mapping-generator-test.js @@ -91,6 +91,7 @@ describe('MappingGenerator', function(){ done(); }); }); + it('recognizes an multi_field and maps it as one', function(done){ generator.generateMapping(new Schema({ test: { @@ -153,6 +154,38 @@ describe('MappingGenerator', function(){ done(); }); }); + it('recognizes a nested array with a simple type and maps it as a simple attribute', function(done){ + generator.generateMapping(new Schema({ + contacts: [String] + }), function(err, mapping){ + mapping.properties.contacts.type.should.eql('string'); + done(); + }); + }); + it('recognizes a nested array with a simple type and additional attributes and maps it as a simple attribute', function(done){ + generator.generateMapping(new Schema({ + contacts: [{ type: String, es_index: 'not_analyzed' }] + }), function(err, mapping){ + mapping.properties.contacts.type.should.eql('string'); + mapping.properties.contacts.index.should.eql('not_analyzed'); + done(); + }); + }); + it('recognizes a nested array with a complex object and maps it', function(done){ + generator.generateMapping(new Schema({ + name: String, + contacts: [{ + email: {type: String, es_index: 'not_analyzed' }, + telephone: String + }] + }), function(err, mapping){ + mapping.properties.name.type.should.eql('string'); + mapping.properties.contacts.properties.email.type.should.eql('string'); + mapping.properties.contacts.properties.email.index.should.eql('not_analyzed'); + mapping.properties.contacts.properties.telephone.type.should.eql('string'); + done(); + }); + }); it('excludes a virtual property from mapping', function(done){ var PersonSchema = new Schema({ first_name: {type: String}, diff --git a/test/models/tweet.js b/test/models/tweet.js old mode 100644 new mode 100755 index f991c3ad..50105f39 --- a/test/models/tweet.js +++ b/test/models/tweet.js @@ -10,6 +10,6 @@ var TweetSchema = new Schema({ , message: String }); -TweetSchema.plugin(mongoosastic) +TweetSchema.plugin(mongoosastic.plugin()) module.exports = mongoose.model('Tweet', TweetSchema); diff --git a/test/search-features-test.js b/test/search-features-test.js old mode 100644 new mode 100755 index 6954b07f..9bfc087b --- a/test/search-features-test.js +++ b/test/search-features-test.js @@ -1,5 +1,4 @@ var mongoose = require('mongoose') - , elastical = require('elastical') , should = require('should') , config = require('./config') , Schema = mongoose.Schema @@ -7,14 +6,13 @@ var mongoose = require('mongoose') , async = require('async') , mongoosastic = require('../lib/mongoosastic'); -var esClient = new elastical.Client(); var BondSchema = new Schema({ name: String , type: {type:String, default:'Other Bond'} , price: Number }); -BondSchema.plugin(mongoosastic); +BondSchema.plugin(mongoosastic.plugin()); var Bond = mongoose.model('Bond', BondSchema); @@ -42,12 +40,10 @@ describe('Query DSL', function(){ describe('range', function(){ it('should be able to find within range', function(done){ Bond.search({ - query:{ - range: { - price:{ - from:20000 - , to: 30000 - } + range: { + price:{ + from:20000 + , to: 30000 } } }, function(err, res){ diff --git a/test/serialize-test.js b/test/serialize-test.js old mode 100644 new mode 100755 index 5eb0c3e5..710ede54 --- a/test/serialize-test.js +++ b/test/serialize-test.js @@ -15,6 +15,7 @@ var PersonSchema22 = new Schema({ }, dob: Date, bowlingBall: {type:Schema.ObjectId, ref:'BowlingBall'}, + games: [{score: Number, date: Date}], somethingToCast : { type: String, es_cast: function(element){ @@ -34,25 +35,40 @@ generator.generateMapping(PersonSchema22, function(err, tmp) { describe('serialize', function(){ var dude = new Person({ - name: {first:'Jeffery', last:'Lebowski'}, + name: {first:'Jeffrey', last:'Lebowski'}, dob: new Date(Date.parse('05/17/1962')), bowlingBall: new BowlingBall(), + games: [{score: 80, date: new Date(Date.parse('05/17/1962'))}, {score: 80, date: new Date(Date.parse('06/17/1962'))}], somethingToCast: 'Something' }); + + // another person with missing parts to test robustness + var millionnaire = new Person({ + name: {first:'Jeffrey', last:'Lebowski'}, + }); + + it('should serialize a document with missing bits', function(){ + var serialized = serialize(millionnaire, mapping); + serialized.should.have.property('games', []); + }); + describe('with no indexed fields', function(){ var serialized = serialize(dude, mapping); it('should serialize model fields', function(){ - serialized.name.first.should.eql('Jeffery'); + serialized.name.first.should.eql('Jeffrey'); serialized.name.last.should.eql('Lebowski'); }); it('should serialize object ids as strings', function(){ serialized.bowlingBall.should.not.eql(dude.bowlingBall); serialized.bowlingBall.should.be.type('string'); }); - it('should serialize dates in ISO 8601 format', function(){ serialized.dob.should.eql(dude.dob.toJSON()) }); + it('should serialize nested arrays', function(){ + serialized.games.should.have.lengthOf(2); + serialized.games[0].should.have.property('score', 80); + }); it('should cast and serialize field', function(){ serialized.somethingToCast.should.eql('Something has been cast') diff --git a/test/synchronize-test.js b/test/synchronize-test.js old mode 100644 new mode 100755 index f7b42237..776221d7 --- a/test/synchronize-test.js +++ b/test/synchronize-test.js @@ -1,6 +1,4 @@ var mongoose = require('mongoose') - , elastical = require('elastical') - , esClient = new(require('elastical').Client) , should = require('should') , config = require('./config') , Schema = mongoose.Schema @@ -11,7 +9,7 @@ var mongoose = require('mongoose') var BookSchema = new Schema({ title: String }); -BookSchema.plugin(mongoosastic); +BookSchema.plugin(mongoosastic.plugin()); var Book = mongoose.model('Book', BookSchema); @@ -47,7 +45,7 @@ describe('Synchronize', function(){ stream.on('close', function(){ count.should.eql(53); setTimeout(function(){ - Book.search({query:'American'}, function(err, results){ + Book.search({query_string: {query: 'American'}}, function(err, results){ results.hits.total.should.eql(2); done(); }); diff --git a/test/truncate-test.js b/test/truncate-test.js old mode 100644 new mode 100755 index cafcfc20..caa4f923 --- a/test/truncate-test.js +++ b/test/truncate-test.js @@ -1,6 +1,4 @@ var mongoose = require('mongoose'), - elastical = require('elastical'), - esClient = new(require('elastical').Client), should = require('should'), config = require('./config'), Schema = mongoose.Schema, @@ -11,7 +9,7 @@ var mongoose = require('mongoose'), var DummySchema = new Schema({ text: String }); -DummySchema.plugin(mongoosastic); +DummySchema.plugin(mongoosastic.plugin()); var Dummy = mongoose.model('Dummy', DummySchema); @@ -40,11 +38,13 @@ describe('Truncate', function() { after(function(done) { Dummy.remove(done); }); - describe('truncate', function() { + describe('esTruncate', function() { it('should be able to truncate all documents', function(done) { Dummy.esTruncate(function(err) { Dummy.search({ - query: 'Text1' + query_string: { + query: 'Text1' + } }, function(err, results) { results.hits.total.should.eql(0); done(err); @@ -52,4 +52,4 @@ describe('Truncate', function() { }); }); }); -}); \ No newline at end of file +});