Announcement Announcement Module
Collapse
No announcement yet.
Handle big HttpEntity response Page Title Module
Move Remove Collapse
X
Conversation Detail Module
Collapse
  • Filter
  • Time
  • Show
Clear All
new posts

  • Handle big HttpEntity response

    Hi.

    I make a call to my restful controller that receive a very big amount of json data.

    How can I store this json into a file? I want to read in sequence each item using custom jackson object mapper..

    Thank you!
    Marco

  • #2
    This is what I used but be aware it might crash some time.

    (function() {
    app = app || {};
    app.ui = {};

    var ROOT_DIR = Ti.Filesystem.applicationDataDirectory;
    var SEP = Titanium.Filesystem.getSeparator();

    // Initialize database
    var db = Titanium.Database.install('contentTest.sqlite','co ntentTest');
    db.execute('DROP TABLE IF EXISTS test');
    db.execute('CREATE TABLE IF NOT EXISTS test (test_id INTEGER PRIMARY KEY, test_content BLOB)');
    db.close();


    var countTest = 100;


    app.ui.createDBFileTestWindow = function(){
    var win = Titanium.UI.createWindow();

    var view = Titanium.UI.createView({backgroundColor:'#fff', layout: 'vertical'});

    var content = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaa';


    var TestFileCallback = function(count){

    var filePath = ROOT_DIR + SEP + 'testFile-' + count + '.txt';
    var f = Titanium.Filesystem.getFile(filePath);

    f.write(content);
    f = null;

    f = Titanium.Filesystem.getFile(filePath);
    var blob = f.read();
    var readText = blob.text;

    f = null;
    blob = null;

    return readText;
    }



    var testDBCallback = function(count){

    var db = Titanium.Database.open('contentTest');
    var theData = db.execute('INSERT INTO test (test_id, test_content) VALUES(?,?)',count,content);

    var resultset = db.execute('SELECT test_content FROM test WHERE test_id = ' + count);
    var readText = resultset.fieldByName("test_content");

    resultset.close();

    db = null;
    theData = null;
    resultset = null;


    return readText;

    }

    var buttonFile = Titanium.UI.createButton({title: 'Test File',top: 10,width: 100,height: 50});
    buttonFile.addEventListener('click',function(e){
    Titanium.API.info("You clicked the File button");
    var startTime = new Date().getTime();
    var content = null;

    var fileCounter = 0;
    for (i=0;i<countTest;i++){
    var content = TestFileCallback(i);
    fileCounter++;
    }

    var endTime = new Date().getTime();
    var diffTime = endTime - startTime;

    for (i=0;i<countTest;i++){
    var filePath = ROOT_DIR + SEP + 'testFile-' + i + '.txt';
    var f = Titanium.Filesystem.getFile(filePath);
    f.deleteFile();
    f = null;
    }

    alert('Count: ' + fileCounter + ' Duration: ' + diffTime + ' ms' + ' Content-Length: ' + content.length);

    });

    var buttonDatabase = Titanium.UI.createButton({title: 'Test Database',top: 10,width: 100,height: 50});
    buttonDatabase.addEventListener('click',function(e ){
    Titanium.API.info("You clicked the Database button");
    var startTime = new Date().getTime();
    var content = null;
    var dbCounter = 0;
    for (i=0;i<countTest;i++){
    content = testDBCallback(i);
    dbCounter++;
    }

    var endTime = new Date().getTime();
    var diffTime = endTime - startTime;

    var db = Titanium.Database.open('contentTest');
    for (i=0;i<countTest;i++){
    theData = db.execute('DELETE FROM test WHERE test_id = ' + i);
    }
    db.close();

    alert('Count: ' + dbCounter + ' Duration: ' + diffTime + ' ms' + ' Content-Length: ' + content.length);
    });

    view.add(buttonFile);
    view.add(buttonDatabase);

    win.add(view);

    return win;
    }

    })();

    Comment


    • #3
      I've solved.

      I save the byte[] of the response by pass a ByteArrayHttpMessageConverter to the RestTemplate and then streaming the result of restTemplate.exchange(url, HttpMethod.GET, requestEntity, byte[].class) to a file.

      Then with jackson I create a parser for the file:
      final JsonParser jp = new ObjectMapper().getJsonFactory().createJsonParser(f ile);

      In this way I can scan sequentially a very big json file avoiding to map it to a List and to get out of memory exceptions..

      In my example this json file contains ~ 56000 orders, with about 10 each field.

      Hope this help someone!
      Marco

      Comment

      Working...
      X