Project

General

Profile

« Previous | Next » 

Revision 58819

Added by Dimitris Pierrakos almost 4 years ago

Handle large json files

View differences:

PiwikDownloadLogs.java
103 103 
                    outFolder = repoLogsPath;
104 104 
                }
105 105 
                FileSystem fs = FileSystem.get(new Configuration());
106 
                FSDataOutputStream fin = fs.create(new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) + ".json"), true);
106 
                //FSDataOutputStream fin = fs.create(new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) + ".json"), true);
107 107 

  		




  108
  108
  
    
                String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + siteId + period + format + "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;


  		




  109
  109
  
    
                String content = "";


  		




  ......




  111
  111
  
    
                int i = 0;


  		




  112
  112
  
    

  		




  113
  113
  
    
                while (!content.equals("[]\n")) {


  		




  
  114
  
    
                    FSDataOutputStream fin = fs.create(new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) +  "_"+i+".json"), true);


  		




  114
  115
  
    
                    String apiUrl = baseApiUrl;


  		




  115
  116
  
    

  		




  116
  117
  
    
                    if (i > 0) {


  		




  ......




  122
  123
  
    
                    fin.write(content.getBytes());


  		




  123
  124
  
    

  		




  124
  125
  
    
                    i++;


  		




  
  126
  
    
                    fin.close();


  		




  125
  127
  
    
                }


  		




  126
  
  
    
                fin.close();


  		




  
  128
  
    
                //fin.close();


  		




  127
  129
  
    

  		




  128
  130
  
    
            }


  		




  129
  131
  
    

  		




  130
  132
  
    
        }


  		




  
  133
  
    
    }


  		




  131
  134
  
    
}


  		












Also available in:  Unified diff