If possible, you should prefer binary over text formats - binary formats have problems of their own (like endianness), but still, as mentioned by SPWorley above, parsing time is dominating this task, and you don’t have to parse much with binary formats. But if you have to deal with textual mesh formats, then you should definitely re-examine your parsing code - no way parsing 160MB could take 2 minutes on contemporary machine. I looked quickly into OFF file format, and come up with some quick tests - see attached files (I know the code is rather ugly, in many respects, but it should do for quick test). The file foo.c is generating 4M vertices in ~140MB OFF file, and then files bar1.c and bar2.cpp are parsing this format, the first one directly from disk, and the later by loading the whole file in memory, and then parsing from there. Timings on my machine are, for first version of reader:
[user@host tmp]$ gcc -o bar1 -Wall bar1.c && time ./bar1 foo.txt
real 0m3.347s
user 0m3.246s
sys 0m0.100s
and for second version of reader:
[user@host tmp]$ g++ -o bar2 -Wall bar2.cpp && time ./bar2 foo.txt
real 0m0.632s
user 0m0.309s
sys 0m0.321s
I know OFF format is actually more complicated than these simple parsers are able to handle, but still it’s impossible to take minutes to load, and also you could see that everything is better than scanning file line-by-line. Of course, bar2.cpp code is really stupid, and there exist many better ways to improve parser; as pointed by SPWorley, I was wrong in my previous message, and you could even try to employ CUDA for this (albeit, as OFF format is somewhat irregular, I don’t think that this would help much here; but I guess this approach could be of big help on those record-based textual file formats, usually created by some old Fortran codes…).
EDIT: Hmm, seems like I’m really unable to make attachment to work on this forum,so here are mentioned files in-line:
foo.c
#include <stdio.h>
#define NVERT 4000000
int
main(int argc, char **argv)
{
FILE *file;
int i;
if (argc < 2)
return 0;
file = fopen(argv[1], "w");
fprintf(file, "OFF\n");
fprintf(file, "%d %d %d\n", NVERT, NVERT / 3, 0);
for (i = 0; i < NVERT; ++i)
fprintf(file, "%f %f %f\n", 0.123456f, 0.123456f, 0.123456f);
for (i = 0; i < NVERT / 3; ++i)
fprintf(file, "%d %d %d %d\n", 3, 3 * i, 3 * i + 1, 3 * i + 2);
fclose(file);
return 0;
}
bar1.c
#include <stdio.h>
#include <stdlib.h>
int
main(int argc, char **argv)
{
FILE *file;
char header[3];
int nvert;
float *pvert;
int ntri;
int *ntrivert;
float **ptrivert;
int i;
int j;
if (argc < 2)
return 0;
file = fopen(argv[1], "r");
fscanf(file, "%3s", header);
fscanf(file, "%d%d", &nvert, &ntri);
pvert = (float *) malloc(nvert * 3 * sizeof(float));
for (i = 0; i < nvert; ++i)
fscanf(file, "%f%f%f", &pvert[3 * i], &pvert[3 * i + 1],
&pvert[3 * i + 2]);
ntrivert = (int *) malloc(ntri * sizeof(int));
ptrivert = (float **) malloc(ntri * sizeof(float *));
for (i = 0; i < ntri; ++i) {
fscanf(file, "%d", &ntrivert[i]);
ptrivert[i] = (float *) malloc(ntrivert[i] * sizeof(float));
for (j = 0; j < ntrivert[i]; ++j)
fscanf(file, "%f", &ptrivert[i][j]);
}
fclose(file);
/* here, do something with the mesh */
free(pvert);
free(ntrivert);
for (i = 0; i < ntri; ++i)
free(ptrivert[i]);
free(ptrivert);
return 0;
}
bar2.cpp
#include <fstream>
#include <sstream>
#include <string>
int
main(int argc, char **argv)
{
if (argc < 2)
return 0;
std::ifstream file(argv[1]);
file.seekg(0, std::ios::end);
size_t size = file.tellg();
file.seekg(0, std::ios::beg);
char *buffer = new char;
file.read(buffer, size);
buffer = 0;
std::istringstream stream(buffer);
std::string header;
stream >> header;
int nvert;
int ntri;
stream >> nvert >> ntri;
float *pvert = new float[3 * nvert];
for (int i = 0; i < 3; ++i)
stream >> pvert[3 * i] >> pvert[3 * i + 1] >> pvert[3 * i + 2];
int *ntrivert = new int[ntri];
float **ptrivert = new float *[ntri];
for (int i = 0; i < ntri; ++i) {
stream >> ntrivert[i];
ptrivert[i] = new float[ntrivert[i]];
for (int j = 0; j < ntrivert[i]; ++j)
stream >> ptrivert[i][j];
}
delete[]buffer;
/* here, do something with the mesh */
delete[]pvert;
delete[]ntrivert;
for (int i = 0; i < ntri; ++i)
delete[]ptrivert[i];
delete[]ptrivert;
return 0;
}