ElasticSearch returning error when trying to insert null to ingest-attachment field

Question

I have installed ingest-attachment processor and am reading file path from one index documents and indexing the file contents in another index documents_attachment using java code.

In this process, if the file is available it will decode into base64 and those contents is attached to a json field fileContent and index those field in another index documents_attachment .

If the file is not available, am trying to attach null as value to the json field fileContent and am trying to index those field also. During this process am getting the below error, when i trying to insert null to the json field fileContent .

Please find the error below.

ElasticsearchStatusException[Elasticsearch exception [type=exception, reason=java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [fileContent] is null, cannot parse.]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=java.lang.IllegalArgumentException: field [fileContent] is null, cannot parse.]]; nested: ElasticsearchException[Elasticsearch exception [type=illegal_argument_exception, reason=field [fileContent] is null, cannot parse.]];
    at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:177)
    at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:573)
    at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:549)
    at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:456)
    at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:429)
    at org.elasticsearch.client.RestHighLevelClient.index(RestHighLevelClient.java:312)
    at com.es.utility.DocumentIndex.main(DocumentIndex.java:193)
    Suppressed: org.elasticsearch.client.ResponseException: method [PUT], host [http://localhost:9200], URI [/document_attachment_dev/doc/129439?pipeline=document_attachment_dev&timeout=1m], status line [HTTP/1.1 500 Internal Server Error]
{"error":{"root_cause":[{"type":"exception","reason":"java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [fileContent] is null, cannot parse.","header":{"processor_type":"attachment"}}],"type":"exception","reason":"java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [fileContent] is null, cannot parse.","caused_by":{"type":"illegal_argument_exception","reason":"java.lang.IllegalArgumentException: field [fileContent] is null, cannot parse.","caused_by":{"type":"illegal_argument_exception","reason":"field [fileContent] is null, cannot parse."}},"header":{"processor_type":"attachment"}},"status":500}

Please find my java code.

public class DocumentIndex {

    private final static String INDEX = "documents_local";  
    private final static String ATTACHMENT = "document_attachment"; 
    private final static String TYPE = "doc";
    private static final Logger logger = Logger.getLogger(Thread.currentThread().getStackTrace()[0].getClassName());

    public static void main(String args[]) throws IOException {


        RestHighLevelClient restHighLevelClient = null;
        Document doc=new Document();

        logger.info("Started Indexing the Document.....");

        try {
            restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("localhost", 9200, "http"),
                    new HttpHost("localhost", 9201, "http")));
        } catch (Exception e) {
            System.out.println(e.getMessage());
        }


        //Fetching Id, FilePath & FileName from Document Index. 
        SearchRequest searchRequest = new SearchRequest(INDEX); 
        searchRequest.types(TYPE);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        QueryBuilder qb = QueryBuilders.matchAllQuery();
        searchSourceBuilder.query(qb);
        searchSourceBuilder.size(3000);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = null;
        try {
             searchResponse = restHighLevelClient.search(searchRequest);
        } catch (IOException e) {
            e.getLocalizedMessage();
        }

        SearchHit[] searchHits = searchResponse.getHits().getHits();
        long totalHits=searchResponse.getHits().totalHits;
        logger.info("Total Hits --->"+totalHits);

        int line=1;

        Map<String, Object> jsonMap ;
        for (SearchHit hit : searchHits) {

            String encodedfile = null;
            File file=null;

            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            doc.setId((int) sourceAsMap.get("id"));
            doc.setApp_language(sourceAsMap.get("app_language").toString());


            String filepath=doc.getPath().concat(doc.getFilename());

            logger.info("Line Number--> "+line+++"ID---> "+doc.getId()+"File Path --->"+filepath);

            try(PrintWriter out = new PrintWriter(new FileOutputStream(new File("d:\\AllFilePath.txt"), true))  ){
                out.println("Line Number--> "+line+"ID---> "+doc.getId()+"File Path --->"+filepath);
            }

            file = new File(filepath);
            if(file.exists() && !file.isDirectory()) {
                try {
                      try(PrintWriter out = new PrintWriter(new FileOutputStream(new File("d:\\AvailableFile.txt"), true))  ){
                            out.println("Line Number--> "+line+++"ID---> "+doc.getId()+"File Path --->"+filepath);
                        }
                    FileInputStream fileInputStreamReader = new FileInputStream(file);
                    byte[] bytes = new byte[(int) file.length()];
                    fileInputStreamReader.read(bytes);
                    encodedfile = new String(Base64.getEncoder().encodeToString(bytes));
                } catch (FileNotFoundException e) {
                    e.printStackTrace();
                }
            }

            jsonMap = new HashMap<>();
            jsonMap.put("id", doc.getId());
            jsonMap.put("app_language", doc.getApp_language());
            jsonMap.put("fileContent", encodedfile); // inserting null here when file is not available and it is not able to encoded.

            String id=Long.toString(doc.getId());

            IndexRequest request = new IndexRequest(ATTACHMENT, "doc", id )
                    .source(jsonMap)
                    .setPipeline(ATTACHMENT);

            PrintStream printStream = new PrintStream(new File("d:\\exception.txt"));
            try {
                IndexResponse response = restHighLevelClient.index(request);

            } catch(ElasticsearchException e) {
                if (e.status() == RestStatus.CONFLICT) {
                }
                e.printStackTrace(printStream);
            }

            line++;

        }

        logger.info("Indexing done.....");

    }

}

Please find my mapping details

PUT _ingest/pipeline/document_attachment
    {
      "description" : "Extract attachment information",
      "processors" : [
        {
          "attachment" : {
            "field" : "fileContent"
          }
        }
      ]
    }

PUT document_attachment
{

     "settings": {
     "analysis": {
      "analyzer": {
        "custom_analyzer": {
          "type": "custom",
          "tokenizer": "whitespace",
          "char_filter": [
            "html_strip"
          ],
          "filter": [
            "lowercase",
            "asciifolding"
          ]
        },
        "product_catalog_keywords_analyzer": {
          "type": "custom",
          "tokenizer": "whitespace",
          "char_filter": [
            "html_strip"
          ],
          "filter": [
            "lowercase",
            "asciifolding"
          ]
        }
      }
    }
  },

  "mappings" : {
    "doc" : {
      "properties" : {
        "attachment" : {
          "properties" : {
            "content" : {
              "type" : "text",
              "analyzer": "custom_analyzer"
            },
            "content_length" : {
              "type" : "long"
            },
            "content_type" : {
              "type" : "text"
            },
            "language" : {
              "type" : "text"
            }
          }
        },
        "fileContent" : {
          "type" : "text"
        },
        "id": {
        "type": "long"
        },
        "app_language" : {
        "type" : "text"
        },

      }
    }
  }
}

Answer 1

Am using the below mapping configuration for ingest-attachment processor and it is working file when the file content is not available (null).

PUT _ingest/pipeline/document_attachment
    {
  "description" : "my first pipeline with handled exceptions",
  "processors" : [
    {
      "attachment" : {
        "field" : "fileContent",
        "on_failure" : [
          {
            "set" : {
              "field" : "error",
              "value" : "{{ _ingest.on_failure_message }}"
            }
          }
        ]
      }
    }
  ]
}