Elasticsearch in laravel 5.1

Question

I want to integrate elasticsearch in my laravel project.

I have installed using following line :

Run command on terminal :

composer require shift31/laravel-elasticsearch:~1.0

Then i have created elasticsearch.php in app/config/ and added following code.

<?php

use Monolog\Logger;

return array(
    'hosts' => array(
                    'your.elasticsearch.server:9200' // what should be my host ?
                    ),
    'logPath' => 'path/to/your/elasticsearch/log',
    'logLevel' => Logger::INFO
);

My first question : What should i write in place of host name

Right now my project is running on local server with localhost:8000.

I have added Shift31\LaravelElasticsearch\ElasticsearchServiceProvider in app/config/app.php for enable the 'Es' facade.

Above all things done. Now in which file i should add the code of elasticsearch to add, update, delete and search the records.

I have product table I need to add product records in elasticsearch, when update product, records should be update.

I have no idea of the further process. Please guide me I have searched on google but no any example help me.

Answer 1

Create the following helper classes in their respective paths:

App\Traits\ElasticSearchEventTrait.php

<?php

Namespace App\Traits;

trait ElasticSearchEventTrait {
    public $esRemoveDefault = array('created_at','updated_at','deleted_at');

    public static function boot()
    {
        parent::boot();

        static::bootElasticSearchEvent();
    }

    public static function bootElasticSearchEvent()
    {
        static::created(function ($model) {
            if(isset($model->esEnabled) && $model->esEnabled === true)
            {
                $model->esCreate();
            }
        });

        static::updated(function ($model) {
            if(isset($model->esEnabled) && $model->esEnabled === true)
            {            
                $model->esUpdate();
            }
        });

        static::deleted(function ($model) {
            if(isset($model->esEnabled) && $model->esEnabled === true)
            {
                $model->esUpdate();
            }
        });
    }

    private function esCreate()
    {
        //esContext is false for polymorphic relations with no elasticsearch indexing
        if(isset($this->esMain) && $this->esMain === true && $this->esContext !== false)
        {
            \Queue::push('ElasticSearchHelper@indexTask',array('id'=>$this->esGetId(),'class'=>get_class($this),'context'=>$this->esGetContext(),'info-context'=>$this->esGetInfoContext(),'excludes'=>$this->esGetRemove()));
        }
        else
        {
            $this->esUpdate();
        }
    }

    private function esUpdate()
    {
        //esContext is false for polymorphic relations with no elasticsearch indexing
        if($this->esContext !== false)
        {
            \Queue::push('ElasticSearchHelper@updateTask',array('id'=>$this->esGetId(),'class'=>get_class($this),'context'=>$this->esGetContext(),'info-context'=>$this->esGetInfoContext(),'excludes'=>$this->esGetRemove()));
        }
    }

    /*
     * Get Id of Model
     */
    public function esGetId()
    {
        if(isset($this->esId))
        {
            return $this->esId;
        }
        else
        {
            return $this->id;
        }
    }

    public function esGetInfoContext()
    {
        if(isset($this->esInfoContext))
        {
            return $this->esInfoContext;
        }
        else
        {
            throw new \RuntimeException("esInfoContext attribute or esGetInfoContext() is not set in class '".get_class($this)."'");
        }
    }

    /*
     * Name of main context of model
     */
    public function esGetContext()
    {
        if(isset($this->esContext))
        {
            return $this->esContext;
        }
        else
        {
            throw new \RuntimeException("esContext attribute or esGetContext() method must be set in class '".get_class($this)."'");
        }
    }

    /*
     * All attributes that needs to be removed from model
     */
    public function esGetRemove()
    {
        if(isset($this->esRemove))
        {
            return array_unique(array_merge($this->esRemoveDefault,$this->esRemove));
        }
        else
        {
            return $this->esRemoveDefault;
        }
    }

    /*
     * Extends Illuminate Collection to provide additional array functions
     */
    public function newCollection(array $models = Array())
    {
        return new Core\Collection($models);
    }

    /**
     * Return a timestamp as DateTime object.
     *
     * @param  mixed  $value
     * @return \Carbon\Carbon
     */
    public function asEsDateTime($value)
    {
            // If this value is an integer, we will assume it is a UNIX timestamp's value
            // and format a Carbon object from this timestamp. This allows flexibility
            // when defining your date fields as they might be UNIX timestamps here.
            if (is_numeric($value))
            {
                    return \Carbon::createFromTimestamp($value);
            }

            // If the value is in simply year, month, day format, we will instantiate the
            // Carbon instances from that format. Again, this provides for simple date
            // fields on the database, while still supporting Carbonized conversion.
            elseif (preg_match('/^(\d{4})-(\d{2})-(\d{2})$/', $value))
            {
                    return \Carbon::createFromFormat('Y-m-d', $value)->startOfDay();
            }

            // Finally, we will just assume this date is in the format used by default on
            // the database connection and use that format to create the Carbon object
            // that is returned back out to the developers after we convert it here.
            elseif ( ! $value instanceof DateTime)
            {
                    $format = $this->getEsDateFormat();

                    return \Carbon::createFromFormat($format, $value);
            }

            return \Carbon::instance($value);
    }

    /**
     * Get the format for database stored dates.
     *
     * @return string
     */
    private function getEsDateFormat()
    {
            return $this->getConnection()->getQueryGrammar()->getDateFormat();
    }

    /*
     * Converts model to a suitable format for ElasticSearch
     */
    public function getEsSaveFormat()
    {
        $obj = clone $this;

        //Go through ES Accessors
        \ElasticSearchHelper::esAccessor($obj);

        $dates = $this->getDates();
        //Convert to array, then change Date to appropriate Elasticsearch format.
        //Why? Because eloquent's date accessors is playing me.
        $dataArray = $obj->attributesToArray();

        //Remove all Excludes
        foreach($this->esGetRemove() as $ex)
        {
            if(array_key_exists($ex,$dataArray))
            {
                unset($dataArray[$ex]);
            }
        }

        if(!empty($dates))
        {
            foreach($dates as $d)
            {
                if(isset($dataArray[$d]) && $dataArray[$d] !== "" )
                {
                    //Trigger Eloquent Getter which will provide a Carbon instance
                    $dataArray[$d] = $this->{$d}->toIso8601String();
                }
            }
        }

        return $dataArray;
    }
}

App\Services\ElasticServiceHelper.php

<?php
/**
 * Description of ElasticSearchHelper: Helps with Indexing/Updating with Elastic Search Server (https://www.elastic.co)
 *
 * @author kpudaruth
 */

Namespace App\Services;

class ElasticSearchHelper {

    /*
     * Laravel Queue - Index Task
     * @param array $job
     * @param array $data
     */
    public function indexTask($job,$data)
    {
        if(\Config::get('website.elasticsearch') === true)
        {
            if(isset($data['context']))
            {
                $this->indexEs($data);
            }
            else
            {
                \Log::error('ElasticSearchHelper: No context set for the following dataset: '.json_encode($data));
            }
        }
        $job->delete();
    }

    /*
     * Laravel Queue - Update Task
     * @param array $job
     * @param array $data
     */
    public function updateTask($job,$data)
    {
        if(\Config::get('website.elasticsearch') === true)
        {
            if(isset($data['context']))
            {
                $this->updateEs($data);
            }
            else
            {
                \Log::error('ElasticSearchHelper: No context set for the following dataset: '.json_encode($data));
            }
        }
        $job->delete();
    }

    /*
     * Index Elastic Search Document
     * @param array $data
     */
    public function indexEs($data)
    {
        $params = array();
        $params['index'] = \App::environment();
        $params['type'] = $data['context'];
        $model = new $data['class'];
        $form = $model::find($data['id']);
        if($form)
        {
            $params['id'] = $form->id;
            if($form->timestamps)
            {
                $params['timestamp'] = $form->updated_at->toIso8601String();
            }
            $params['body'][$data['context']] = $this->saveFormat($form);
            \Es::index($params);
        }
    }

    /*
     * Update Elastic Search
     * @param array $data
     */
    public function updateEs($data)
    {
        $params = array();
        $params['index'] = \App::environment();
        $params['type'] = $data['context'];
        $model = new $data['class'];
        $form = $model::withTrashed()->find($data['id']);

        if(count($form))
        {
            /*
             * Main form is being updated
             */
            if($data['info-context'] === $data['context'])
            {
                $params['id'] = $data['id'];
                $params['body']['doc'][$data['info-context']] = $this->saveFormat($form);
            }
            else
            {
                //Form is child, we get parent
                $parent = $form->esGetParent();
                if(count($parent))
                {
                    //Id is always that of parent
                    $params['id'] = $parent->id;
                    //fetch all children, given that we cannot save per children basis
                    $children = $parent->{$data['info-context']}()->get();
                    if(count($children))
                    {
                        //Get data in a format that can be saved by Elastic Search
                        $params['body']['doc'][$data['info-context']] = $this->saveFormat($children);
                    }
                    else
                    {
                        //Empty it is
                        $params['body']['doc'][$data['info-context']] = array();
                    }
                }
                else
                {
                    \Log::error("Parent not found for {$data['context']} - {$data['class']}, Id: {$data['id']}");
                    return false;
                }
            }

            //Check if Parent Exists
            try
            {
                $result = \Es::get([
                    'id' => $params['id'],
                    'index' => $params['index'],
                    'type' => $data['context']
                ]);
            } catch (\Exception $ex) {
                if($ex instanceof \Elasticsearch\Common\Exceptions\Missing404Exception || $ex instanceof \Guzzle\Http\Exception\ClientErrorResponseException)
                {
                    //if not, we set it
                    if (isset($parent) && $parent)
                    {
                        $this->indexEs([
                            'context' => $data['context'],
                            'class' => get_class($parent),
                            'id' => $parent->id,
                        ]);
                    }
                    else
                    {
                        \Log::error('Unexpected error in updating elasticsearch records, parent not set with message: '.$ex->getMessage());
                        return false;
                    }
                }
                else
                {
                    \Log::error('Unexpected error in updating elasticsearch records: '.$ex->getMessage());
                    return false;
                }
            }

            \Es::update($params);
        }
    }

    /*
     * Iterate through all Es accessors of the model.
     * @param \Illuminate\Database\Eloquent\Model $object
     */
    public function esAccessor(&$object)
    {
        if(is_object($object))
        {
            $attributes = $object->getAttributes();
            foreach($attributes as $name => $value)
            {
                $esMutator = 'get' . studly_case($name) . 'EsAttribute';
                if (method_exists($object, $esMutator)) {
                    $object->{$name} = $object->$esMutator($object->{$name});
                }
            }
        }
        else
        {
            throw New \RuntimeException("Expected type object");
        }
    }

    /*
     * Iterates over a collection applying the getEsSaveFormat function
     * @param mixed $object
     *
     * @return array
     */
    public function saveFormat($object)
    {
        if($object instanceof \Illuminate\Database\Eloquent\Model)
        {
            return $object->getEsSaveFormat();
        }
        else
        {
            return array_map(function($value)
            {
                return $value->getEsSaveFormat();
            }, $object->all());
        }
    }
}

A couple of gotchas from the above helper classes:

The default ElasticSearch index is set to the name of the App's Environment

The ..task() functions are meant for the old laravel 4.2 queue format. I've yet to port those to laravel 5.x. Same goes for the Queue::push commands.

Example

ElasticSearch Mapping:

[
    'automobile' => [
        "dynamic" => "strict",
        'properties' => [
            'automobile' => [
                'properties' => [
                    'id' => [
                        'type' => 'long',
                        'index' => 'not_analyzed'
                    ],
                    'manufacturer_name' => [
                        'type' => 'string',
                    ],
                    'manufactured_on' => [
                        'type' => 'date'
                    ]
                ]                      
            ],
            'car' => [
                'properties' => [
                    'id' => [
                        'type' => 'long',
                        'index' => 'not_analyzed'
                    ],
                    'name' => [
                        'type' => 'string',
                    ],
                    'model_id' => [
                        'type' => 'string'
                    ]
                ]                
            ],        
            "car-model" => [
                'properties' => [
                    'id' => [
                        'type' => 'long',
                        'index' => 'not_analyzed'
                    ],
                    'description' => [
                        'type' => 'string',
                    ],
                    'name' => [
                        'type' => 'string'
                    ]
                ]
            ]
        ]
    ]
]

Top level document is called 'automobile'. Underneath it, you have 'automobile', 'car' & 'car-model'. Consider 'car' & 'car-model' as relations to the automobile. They are known as sub documents on elasticsearch. (See: https://www.elastic.co/guide/en/elasticsearch/guide/current/document.html)

Model: App\Models\Car.php

namespace App\Models;

class Car extends \Eloquent {

    use \Illuminate\Database\Eloquent\SoftDeletingTrait;
    use \App\Traits\ElasticSearchEventTrait;

    protected $table = 'car';

    protected $fillable = [
        'name',
        'serie',
        'model_id',
        'automobile_id'
    ];

    protected $dates = [
        'deleted_at'
    ];

    /* Elastic Search */

    //Indexing Enabled
    public $esEnabled = true;
    //Context for Indexing - Top Level name in the mapping
    public $esContext = "automobile";
    //Info Context - Secondary level name in the mapping. 
    public $esInfoContext = "car";
    //The following fields will not be saved in elasticsearch.
    public $esRemove = ['automobile_id'];

    //Fetches parent relation of car, so that we can retrieve its id for saving in the appropriate elasticsearch record
    public function esGetParent()
    {
        return $this->automobile;
    }

    /*
     * Event Observers
     */

    public static function boot() {
        parent:: boot();

        //Attach events to model on start
        static::bootElasticSearchEvent();
    }

    /*
    * ElasticSearch Accessor
    * 
    * Sometimes you might wish to format the data before storing it in elasticsearch,
    * The accessor name is in the format of: get + attribute's name camel case + EsAttribute
    * The $val parameter will always be the value of the attribute that is being accessed.
    *
    * @param mixed $val
    */

    /*
    * Elasticsearch Accessor: Model Id
    *
    * Get the model name and save it
    *
    * @param int $model_id
    * @return string
    */

    public function getModelIdEsAttribute($model_id) {
        //Fetch model from table
        $model = \App\Models\CarModel::find($model_id);
        if($model) {
            //Return name of model if found
            return $model->name;
        } else {
            return '';
        }
    }

    /*
    * Automobile Relationship: Belongs To
    */
    public function automobile()
    {
        return $this->belongsTo('\App\Models\Automobile','automobile_id');
    }

}

Example of Search Query:

/**
* Get search results
* 
* @param string $search (Search string)
* 
*/
public function getAll($search)
{
    $params = array();
    $params['index'] = App::environment();
    //Declare your mapping names in the array which you wish to search on.
    $params['type'] = array('automobile');

    /*
     * Build Query String
     */

    //Exact match is favored instead of fuzzy ones
    $params['body']['query']['bool']['should'][0]['match']['name']['query'] = $search;
    $params['body']['query']['bool']['should'][0]['match']['name']['operator'] = "and";
    $params['body']['query']['bool']['should'][0]['match']['name']['boost'] = 2;
    $params['body']['query']['bool']['should'][1]['fuzzy_like_this']['like_text'] = $search;
    $params['body']['query']['bool']['should'][1]['fuzzy_like_this']['fuzziness'] = 0.5;
    $params['body']['query']['bool']['should'][1]['fuzzy_like_this']['prefix_length'] = 2;
    $params['body']['query']['bool']['minimum_should_match'] = 1;
    //Highlight matches
    $params['body']['highlight']['fields']['*'] = new \stdClass();
    $params['body']['highlight']['pre_tags'] = array('<b>');
    $params['body']['highlight']['post_tags'] = array('</b>');
    //Exclude laravel timestamps
    $params['body']['_source']['exclude'] = array( "*.created_at","*.updated_at","*.deleted_at");

    /*
     * Poll search server until we have some results
     */
    $from_offset = 0;
    $result = array();

    //Loop through all the search results
    do
    {
        try
        {
            $params['body']['from'] = $from_offset;
            $params['body']['size'] = 5;
            $queryResponse = \Es::search($params);
            //Custom function to process the result
            //Since we will receive a bunch of arrays, we need to reformat the data and display it properly.
            $result = $this->processSearchResult($queryResponse);
            $from_offset+= 5;
        }
        catch (\Exception $e)
        {
            \Log::error($e->getMessage());
            return Response::make("An error occured with the search server.",500);
        }

    }
    while (count($result) === 0  && $queryResponse['hits']['total'] > 0);

    echo json_encode($result);
}

/*
* Format search results as necessary
* @param array $queryResponse
*/ 
private function processSearchResult(array $queryResponse)
{
    $result = array();
    //Check if we have results in the array
    if($queryResponse['hits']['total'] > 0 && $queryResponse['timed_out'] === false)
    {
        //Loop through each result
        foreach($queryResponse['hits']['hits'] as $line)
        {
            //Elasticsearch will highlight the relevant sections in your query in an array. The below creates a readable format with · as  delimiter.
            $highlight = "";
            if(isset($line['highlight']))
            {
                foreach($line['highlight'] as $k=>$v)
                {
                    foreach($v as $val)
                    {
                        $highlight[] =  str_replace("_"," ",implode(" - ",explode(".",$k)))." : ".$val;
                    }
                }
                $highlight = implode(" · ",$highlight);
            }

            //Check the mapping type
            switch($line['_type'])
            {
                case "automobile":
                    $result[] = array('icon'=>'fa-automobile',
                                      'title'=> 'Automobile',
                                      'id' => $line['_id'],
                                      //name to be displayed on my search result page
                                      'value'=>$line['_source'][$line['_type']]['name']." (Code: ".$line['_id'].")",
                                      //Using a helper to generate the url. Build your own class.
                                      'url'=>\App\Helpers\URLGenerator::generate($line['_type'],$line['_id']),
                                      //And the highlights as formatted above.
                                      'highlight'=>$highlight);
                    break;
            }

        }
    }

    return $result;
}