代码之家 › 专栏 › 技术社区 › Gary

从客户端和客户端读取的文本文件超过了javascript中单个字符串的最大大小

arraybuffer blob javascript

1

Gary · 技术社区 · 8 年前

事务成功完成后,从blob数组中生成了一个相同类型的新blob。

   try

     {

       let i, l, b, result, map, p;

       const c = 1000000;


       // First get the file map from front of blob/file.

       // Read first ten characters to get length of map JSON string.

       b = new Blob( [ f.slice(0,10) ], { type: 'text/csv' } ); 

       result = await read_file( b );

       l = parseInt(result.value);


       // Read the map string and parse to array of objects.

       b = new Blob( [ f.slice( 10, 10 + l) ], { type: 'text/csv' } ); 

       result = await read_file( b );

       map = JSON.parse(result.value); 


       l = map.length;

       p = 10 + result.value.length;


       // Using this loop taks about 7:30 to complete.

       for ( i = 1; i < l; i++ )

         {

           b = new Blob( [ f.slice( p, p + map[i].l ) ], { type: 'text/csv' } ); 

           result = await read_file( b ); // FileReader wrapped in a promise.

           result = await write_qst( JSON.parse( result.value ) ); // Database transaction wrapped in a promise.

           p = p + map[i].l;

           $("#msg").text( result );

         }; // next i


       $("#msg").text( "Successfully wrote all data to the database." );


       i = l = b = result = map = p = null;

     }

   catch(e)

     { 

       alert( "error " + e );

     }

   finally

     {

       f = null;

     }



/* 

  // Alternative loop that completes in about 1:30 versus 7:30 for above loop.


       for ( i = 1; i < l; i++ )

         { 

           let status = false, 

               k, j, n = 0, x = 0, 

               L = map[i].l,

               a_parse = [];



           if ( L < c ) status = true;

           while ( status )

             {

               if ( i+1 < l && L + map[i+1].l <= c ) 

                 {

                   L = L + map[i+1].l;

                   i = i + 1;

                   n = n + 1;

                 }

               else

                 {

                   status = false;

                 };

             }; // loop while


           b = new Blob( [ f.slice( p, p + L ) ], { type: 'text/csv' } ); 

           result = await read_file( b ); 

           j = i - n; 

           for ( k = j; k <= i; k++ )

             {

                a_parse.push( JSON.parse( result.value.substring( x, x + map[k].l ) ) );

                x = x + map[k].l;

             }; // next k

           result = await write_qst_grp( a_parse, i + ' of ' + l );

           p = p + L;

           $("#msg").text( result );

         }; // next i



*/



/*

// Was using this loop when thought the concern may be that the JSON strings were too large,
// but then realized the issue in my case is the opposite one of having 50,000 JSON strings of smaller size.

       for ( i = 1; i < l; i++ )

         {

           let x,

               m = map[i].l,

               str = [];

           while ( m > 0 )

             {

               x = Math.min( m, c );

               m = m - c;

               b = new Blob( [ f.slice( p, p + x ) ], { type: 'text/csv' } ); 

               result = await read_file( b );

               str.push( result.value );

               p = p + x;

             }; // loop while


            result = await write_qst( JSON.parse( str.join("") ) );

            $("#msg").text( result );

            str = null;

         }; // next i
*/

1 回复 | 直到 8 年前

1

Kaiido 8 年前

.slice() 方法。

function readChunks({blob, chunk_size}) {
  console.log('full Blob size', blob.size);
  const strings = [];  
  const reader = new FileReader();
  var cursor = 0;
  reader.onload = onsingleprocessed;
  
  readNext();
  
  function readNext() {
    // here is the magic
    const nextChunk = blob.slice(cursor, (cursor + chunk_size));
    cursor += chunk_size;
    reader.readAsText(nextChunk);
  }
  function onsingleprocessed() {
    strings.push(reader.result);
    if(cursor < blob.size) readNext();
    else {
      console.log('read %s chunks', strings.length);
      console.log('excerpt content of the first chunk',
        strings[0].substring(0, 30));
    }
  }
}



// we will do the demo in a Worker to not kill visitors page
function worker_script() {
  self.onmessage = e => {
    const blobs = [];
    const chunk_size = 1024*1024; // 1MB per chunk
    for(let i=0; i<500; i++) {
      let arr = new Uint8Array(chunk_size);
      arr.fill(97); // only 'a'
      blobs.push(new Blob([arr], {type:'text/plain'}));
    }
    const merged = new Blob(blobs, {type: 'text/plain'});
    self.postMessage({blob: merged, chunk_size: chunk_size});
  }
}
const worker_url = URL.createObjectURL(
  new Blob([`(${worker_script.toString()})()`],
    {type: 'application/javascript'}
  )
);
const worker = new Worker(worker_url);
worker.onmessage = e => readChunks(e.data);
worker.postMessage('do it');