使用 Apache Camel 从另一个 API 使用 JSON 并将其写入 CSV

Question

I am new in apache camel. I would like to consume json data from another api's end point url (http://localhost:8080/employeePayload). I would like to consume this json data, unmarshal it to a POJO using apache camel and write it to CSV file. Before I write it in CSV I need to add delimiter |" for each property/column. Also I have to add header and trailer records to the CSV file. Header consists of some static column names and some run time names. Header column count will not be same as the json data. Header has 15 column and json has 110 columns).

is there a way to bind the json/pojo to CSV file without using setter method(as I have 110 properties, can't use setter for all)?

Can someone suggest what would be the best way to achieve this?

If headers part is not clear, please tell me how to write the json to CSV with delimiter(|").

Edit: I am able to consume json from another API. However I am not sure how to write it in CSV with delimiter using apache camel or any other lib without using getters and setters.

Answer 1

I found the answer. As the development progressed I had many questions which which I have posted them in stackoverflow.

I decided to sum up everything I found.

Please refer the below post. Unable to add header and footer to the file using Apache Camel Bindy

This solution had fixed many issues that I had in various stage.

Issue 1) Added dynamic header and footer to the flat file using BindyFixedLengthDataFormat. As the name suggests it's fixed length. I had to assign maximum length and trim the whitespaces after generating the body. (I still may need consider using different class as I do not want the length to be fixed). BindyCsvDataFormat does not have the option to add dynamic headers and footer to the file.

Issue 2) I had the requirement to replace the pipe delimiter with |".

Issue 3) Instead of using jetty to expose the end point url which will be hard coded I have used rest configuration so that hostname will no longer be hard coded and you can deploy to any environment.

@Component
    public class EmployeePackageRoute extends RouteBuilder {
    
        @Autowired
        private Header header;
        
        private Trailer footer;
        
        private String file_name;
        
        @SuppressWarnings("deprecation")
        @Override
        public final void configure() {
            
            System.out.println("fetching employee details from employee api");
            
            final SimpleDateFormat TARGET_DATE_FORMAT = new SimpleDateFormat("YYYYMMDDHHMMSS");
            String date = TARGET_DATE_FORMAT.format(new Date());
            
            String year = date.substring(0, 8);
            String hr = date.substring(9, 13);
    
            try {
                restConfiguration().component("jetty").port(8000).bindingMode(RestBindingMode.json);
                rest("/api/test/getTestResponse")
                .get().consumes(MediaType.APPLICATION_JSON_VALUE).route().setBody().constant("return test response");
                
                rest("/api/bcr/recurringPaymentsDeduct")
                .post().consumes(MediaType.APPLICATION_JSON_VALUE).type(RecurringPaymentResults.class).outType(ResponseEntity.class).route()
                .process(
                        ex -> {
                            
                            RecurringPaymentResults result = ex.getIn().getBody(RecurringPaymentResults.class);
                            List<RecurringPaymentsDeduct> employee = result.getResults();
                            file_name = Constants.FILE_NUMBER_DEV+"_"+date+"_"+Constants.AUMBCR_HRMD+result.getSequenceNumber()+"_"+Constants.DUT8G2I+".SAP";
                            header.setHeader7(Constants.FILE_NUMBER_DEV+"_"+date+"_"+Constants.AUMBCR_HRMD+result.getSequenceNumber()+"_"+Constants.DUT8G2I+".SAP");
                            
                            header.setHeader8(year);
                            header.setHeader9(hr);
                            header.setHeader10(Constants.ENVIRONMENT);
                            
                            footer = new Footer("Footer",(employee.size()+2));
                            
                            ex.getOut().setBody(employee);
                        })
                .to("seda:recurringPaymentsFeed");
                
                final DataFormat recurringPaymentsBindy = new BindyFixedLengthDataFormat(RecurringPaymentsDeduct.class);
                
                from("seda:recurringPaymentsFeed")
                .startupOrder(2)
                .log("add footer to the file")
                .process(
                        ex -> {
                            
                            Map<String, Object> headerObjMap = new HashMap<String,Object>();
                            headerObjMap.put(Header.class.getName(), header);
                            System.out.println(" *** file_name 1 - "+file_name);
                            Map<String, Object> footerObjMap = new HashMap<String,Object>();
                            footerObjMap.put(Footer.class.getName(), footer);
                            
                            ex.getOut().setHeader(BindyFixedLengthDataFormat.CAMEL_BINDY_FIXED_LENGTH_HEADER, headerObjMap);
                            ex.getOut().setBody(ex.getIn().getBody());
                            ex.getOut().setHeader(BindyFixedLengthDataFormat.CAMEL_BINDY_FIXED_LENGTH_FOOTER, footerObjMap);
                            
                        })
                .marshal(recurringPaymentsBindy)
                .convertBodyTo(byte[].class, "iso-8859-1")
                .setBody(body().regexReplaceAll("\\|", "\\|\""))
                .log("csv processed")
                .to("file://app/bcr-files?fileName=recurringPaymentsDeduct.SAP")
                .end();
                
                
            } catch(Exception e) {
                System.out.println("Error occurred while processing employee data: "+e.getMessage());
                e.printStackTrace();
            }
        }
        
    }

@Data
@Section(number = 2)
@FixedLengthRecord(header = Header.class, footer = Trailer.class)
public class RecurringPaymentsDeduct implements Serializable {

    private static final long serialVersionUID = 1L;
    
    @DataField(pos = 1, length = 10, delimiter = "|", trim = true)
    public String RECTY;
    
    @DataField(pos = 2, length = 10, delimiter = "|", trim = true)
    public String CLIID;
    
    @DataField(pos = 3, length = 10, delimiter = "|", trim = true)
    public String INTCA;
    
    @DataField(pos = 4, length = 10, delimiter = "|", trim = true)
    public String ORDNO;
    
    @DataField(pos = 5, length = 10, delimiter = "|", trim = true)
    public String IOPER;
    
    @DataField(pos = 6, length = 10, delimiter = "|", trim = true)
    public String INFTY;
    
    @DataField(pos = 7, length = 10, delimiter = "|", trim = true)
    public String SUBTY;
    
    @DataField(pos = 8, length = 10, delimiter = "|", trim = true)
    public String BEGDA;
    
    @DataField(pos = 9, length = 10, delimiter = "|", trim = true)
    public String ENDDA;
    
    @DataField(pos = 10, length = 10, trim = true)
    public String OBJPS;
        
}

@Configuration
@EnableConfigurationProperties
@ConfigurationProperties("adp")
@Data
@Section(number = 1)
@FixedLengthRecord()
public class Header implements Serializable {

    private static final long serialVersionUID = 1L;

    @DataField(pos = 1, length = 5, delimiter = "|", trim = true)
    private String header1;
    
    @DataField(pos = 2, length = 5, delimiter = "|", trim = true)
    private String header2;
    
    @DataField(pos = 3, length = 15, delimiter = "|", trim = true)
    private String header3;
    
    @DataField(pos = 4, length = 60, delimiter = "|", trim = true)
    private String header4;
    
    @DataField(pos = 5, length = 15, delimiter = "|", trim = true)
    private String header5;
    
    @DataField(pos = 6, length = 30, delimiter = "|", trim = true)
    private String header6;
    
    @DataField(pos = 7, length = 30, delimiter = "|", trim = true)
    private String header7;
    
    @DataField(pos = 8, length =  8, pattern = "YYYYMMDD", delimiter = "|", trim = true)
    private String header8;
    
    @DataField(pos = 9, length = 6, delimiter = "|", trim = true)
    private String header9;
    
    @DataField(pos = 10, length = 1, delimiter = "|", trim = true)
    private String header10;
    
    @DataField(pos = 11, length = 2, delimiter = "|", trim = true)
    private String header11;
    
    @DataField(pos = 12, length = 10, delimiter = "|", trim = true)
    private String header12;
    
    @DataField(pos = 13, length = 10, delimiter = "|", trim = true)
    private String header13;
    
    @DataField(pos = 14, length = 10, align = "L", trim = true)
    private String header14;
}

@Data
@Section(number = 3)
@FixedLengthRecord()
public class Footer implements Serializable {

    private static final long serialVersionUID = 1L;

    @DataField(pos = 1, length = 7, delimiter = "|", trim = true)
    private String trailer;
    
    @DataField(pos = 2, length = 5, align = "L", trim = true)
    private int count;
    
    public Footer() {
        
    }
    
    public Footer(String trailer, int count) {
        this.trailer = trailer;
        this.count = count;
    }
}