I was recently tasked to develop a system that would need to write data in the Gigabytes per second range directly to disk. While doing so, I needed to spec out and build a machine that allowed for that type of bandwidth to be persisted as well as find the most optimized filesystem and write method available in C. Once the specs were done and the machine arrived, it was time to determine which filesystem and write method under Linux would give the best results for writing. I initially tested with bonnie++, but soon became aware that the results I was receiving were not a true test of the raid configuration and filesystem, but were instead block writes and reads. For my purposes I need to benchmark the write speed of several different writing types including asynchronous writing, standard IO and generic (open,write,read) IO. To do so, I created a simple writetest program:
#include <aio.h>
#include <time.h>
#include <stdio.h>
#include <malloc.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <math.h>
int nVersionMajor = 1;
int nVersionMinor = 0;
int nVersionBuild = 1;
int64_t mb = 1048576;
char szOutputDir[1024];
long nSize = 0;
int nIteration;
short nUseAio;
short nUseStd;
short nUseGen;
unsigned long long nStdioAvg = 0;
unsigned long long nAioAvg = 0;
unsigned long long nGenAvg = 0;
double nSec = 1000000.0;
char * pBytes;
void stdioWrite(char * filename){
FILE * pFile;
struct timeval tvStart;
struct timeval tvEnd;
int64_t nEnd;
int64_t nStart;
gettimeofday(&tvStart);
nStart = tvStart.tv_sec * 1000000 + tvStart.tv_usec;
pFile = fopen(filename,"w");
if(!pFile){
fprintf(stderr, "[Error] could not open %s for writing.n", filename);
exit(1);
}//end if
fwrite(pBytes, strlen(pBytes), 1, pFile);
fclose(pFile);
gettimeofday(&tvEnd);
nEnd = tvEnd.tv_sec * 1000000 + tvEnd.tv_usec;
nStdioAvg += (nEnd-nStart);
printf("stdio->fopen,fwrite,fclose t%f sec.n", (nEnd-nStart)/nSec);
}//end stdioWrite()
void aioWrite(char * filename){
struct aiocb aio;
int outFile;
struct timeval tvStart;
struct timeval tvEnd;
int64_t nEnd;
int64_t nStart;
gettimeofday(&tvStart);
nStart = tvStart.tv_sec * 1000000 + tvStart.tv_usec;
outFile = open(filename, O_WRONLY|O_CREAT, 0666);
if(outFile == -1){
fprintf(stderr, "[Error] Could not open %s for writing.n", filename);
exit(1);
}//end if
aio.aio_offset = 0;
aio.aio_fildes = outFile;
aio.aio_reqprio = 0;
aio.aio_buf = pBytes;
aio.aio_nbytes = strlen(pBytes);
aio.aio_sigevent.sigev_notify = SIGEV_NONE;
aio_write(&aio);
while(aio_error(&aio) == EINPROGRESS);
close(outFile);
gettimeofday(&tvEnd);
nEnd = tvEnd.tv_sec * 1000000 + tvEnd.tv_usec;
nAioAvg += (nEnd-nStart);
printf("aio->open,aio_write,close t%f sec.n", (nEnd-nStart)/nSec);
}//end aioWrite()
void genWrite(char * filename){
int outFile;
struct timeval tvStart;
struct timeval tvEnd;
int64_t nEnd;
int64_t nStart;
gettimeofday(&tvStart);
nStart = tvStart.tv_sec * 1000000 + tvStart.tv_usec;
outFile = open(filename, O_WRONLY|O_CREAT, 0666);
if(outFile == -1){
fprintf(stderr, "[Error] Could not open %s for writing.n", filename);
exit(1);
}//end if
write(outFile, pBytes, strlen(pBytes));
close(outFile);
gettimeofday(&tvEnd);
nEnd = tvEnd.tv_sec * 1000000 + tvEnd.tv_usec;
nGenAvg += (nEnd-nStart);
printf("gen->open,write,close t%f sec.n", (nEnd-nStart)/nSec);
}//end genWrite()
void displayUsage(){
printf("writetest - GodLikeMouse file write testing Version %d.%d.%dn", nVersionMajor, nVersionMinor, nVersionBuild);
printf("tCopyright 2008 GodLikeMouse (www.GodLikeMouse.com)n");
printf("n");
printf("Usage:n");
printf("tnqpcapd [options]n");
printf("n");
printf("Options:n");
printf("t--output-dir [directory]n");
printf("ttThe directory to write to for testing (default .test).n");
printf("t--i [iterations]n");
printf("ttThe amount of times to write (default 1).n");
printf("t--mb [megabytes to write]n");
printf("ttThe size of the files to write in megabytes.n");
printf("t--b [bytes to write]n");
printf("ttThe size of the files to write in bytes.n");
printf("ttwritten (default /data).n");
printf("t--stdion");
printf("ttUse fopen,fwrite,fclose.n");
printf("t--aion");
printf("ttUse open,aio_write,close.n");
printf("t--genion");
printf("ttUse open,write,close.n");
printf("t--helpn");
printf("ttDisplay this help message.n");
printf("t--versionn");
printf("ttDisplay the version information.n");
printf("n");
}//end displayUsage()
void parseArgs(int argc, char ** argv){
int i;
for(i=0; i<argc; i++){
if(strstr(argv[i], "--output-dir")){
sprintf(szOutputDir, "%s", argv[++i]);
continue;
}//end if
if(strstr(argv[i], "--mb")){
nSize = mb * atol(argv[++i]);
continue;
}//end if
if(strstr(argv[i], "--b")){
nSize = atol(argv[++i]);
continue;
}//end if
if(strstr(argv[i], "--help")){
displayUsage();
exit(0);
}//end if
if(strstr(argv[i], "--i")){
nIteration = atoi(argv[++i]);
continue;
}//end if
if(strstr(argv[i], "--aio")){
nUseAio = 1;
continue;
}//end if
if(strstr(argv[i], "--stdio")){
nUseStd = 1;
continue;
}//end if
if(strstr(argv[i], "--genio")){
nUseGen = 1;
continue;
}//end if
if(strstr(argv[i], "--version")){
printf("writetest - GodLikeMouse file write testing Version %d.%d.%dn", nVersionMajor, nVersionMinor, nVersionBuild);
exit(0);
}//end if
}//end for
}//end parseArgs()
void printSeparator(){
printf("---------------------------------------------n");
}//end printSeparator()
void printAverages(){
double nTemp;
if(nUseStd){
nTemp = ((double)nStdioAvg/nIteration)/nSec;
printf("stdio average write time: t%f sec.n", nTemp);
printf("stdio average throughput: t%.0f bytes/sec ", nSize/nTemp);
printf("%.0f MB/secn", (nSize/nTemp)/mb);
}//end if
if(nUseAio){
nTemp = ((double)nAioAvg/nIteration)/nSec;
printf("aio average write time: t%f sec.n", nTemp);
printf("aio average throughput: t%.0f bytes/sec ", nSize/nTemp);
printf("%.0f MB/secn", (nSize/nTemp)/mb);
}//end if
if(nUseGen){
nTemp = ((double)nGenAvg/nIteration)/nSec;
printf("gen average write time: t%f sec.n", nTemp);
printf("gen average throughput: t%.0f bytes/sec ", nSize/nTemp);
printf("%.0f MB/secn", (nSize/nTemp)/mb);
}//end if
}//end printAverages()
int main(int argc, char ** argv){
int i;
char szFile[2048];
nIteration = 1;
nSize = 1024;
strcpy(szOutputDir, ".test");
nUseStd = 0;
nUseAio = 0;
nUseGen = 0;
parseArgs(argc, argv);
printf("n");
printf("Beginning cycle writen");
printf("Writing %ld bytes, %ld MBn", nSize, (nSize/mb));
printSeparator();
for(i=0; i<nIteration; i++){
if(nUseStd){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "stdio", i);
stdioWrite(szFile);
free(pBytes);
}//end if
if(nUseAio){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "aio", i);
aioWrite(szFile);
free(pBytes);
}//end if
if(nUseGen){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "gen", i);
genWrite(szFile);
free(pBytes);
}//end if
printSeparator();
}//end for
printf("n");
printAverages();
printf("n");
printf("n");
printf("Beginning sequential writen");
printf("Writing %ld bytes, %ld MBn", nSize, (nSize/mb));
printSeparator();
nStdioAvg = 0;
nAioAvg = 0;
nGenAvg = 0;
if(nUseStd){
for(i=0; i<nIteration; i++){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "stdio", i);
stdioWrite(szFile);
free(pBytes);
}//end for
printSeparator();
}//end if
if(nUseAio){
for(i=0; i<nIteration; i++){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "aio", i);
aioWrite(szFile);
free(pBytes);
}//end for
printSeparator();
}//end if
if(nUseGen){
for(i=0; i<nIteration; i++){
pBytes = (char *)malloc(nSize);
memset(pBytes, 'X', nSize);
sprintf(szFile, "%s/%s.%d", szOutputDir, "gen", i);
genWrite(szFile);
free(pBytes);
}//end for
printSeparator();
}//end if
printf("n");
printAverages();
printf("n");
}//end main()
This simple program took in a set of parameters to determine which write methods to use to write, how much data, how many times and where to write it. Using this I began testing all the available free filesystems I could find for Linux. The fastest write speeds were given by XFS with the following make and mount options.
mkfs.xfs -l size=32m -d agcount=4 /dev/sda4 mount /dev/sda4 /data -o noatime,nodiratime,osyncisdsync
The first round metrics I received using this method to test are as follows:
src/writetest –output-dir /data/ –stdio –aio –genio –i 3 –mb 500
Beginning cycle write Writing 524288000 bytes, 500 MB
———————————————
stdio->fopen,fwrite,fclose 1.047127 sec.
aio->open,aio_write,close 5.724001 sec.
gen->open,write,close 2.580516 sec.
———————————————
stdio->fopen,fwrite,fclose 2.063382 sec.
aio->open,aio_write,close 5.954485 sec.
gen->open,write,close 1.351529 sec.
———————————————
stdio->fopen,fwrite,fclose 1.729958 sec.
aio->open,aio_write,close 6.457802 sec.
gen->open,write,close 1.754718 sec.
———————————————
stdio average write time: 1.613489 sec.
stdio average throughput: 324940548 bytes/sec 310 MB/sec
aio average write time: 6.045429 sec.
aio average throughput: 86724693 bytes/sec 83 MB/sec
gen average write time: 1.895588 sec.
gen average throughput: 276583357 bytes/sec 264 MB/sec
Beginning sequential write Writing 524288000 bytes, 500 MB
———————————————
stdio->fopen,fwrite,fclose 1.479412 sec.
stdio->fopen,fwrite,fclose 1.597659 sec.
stdio->fopen,fwrite,fclose 1.851250 sec.
———————————————
aio->open,aio_write,close 6.575862 sec.
aio->open,aio_write,close 7.003166 sec.
aio->open,aio_write,close 6.991679 sec.
———————————————
gen->open,write,close 1.465857 sec.
gen->open,write,close 0.980143 sec.
gen->open,write,close 0.841066 sec.
———————————————
stdio average write time: 1.642774 sec.
stdio average throughput: 319148043 bytes/sec 304 MB/sec
aio average write time: 6.856902 sec.
aio average throughput: 76461349 bytes/sec 73 MB/sec
gen average write time: 1.095689 sec.
gen average throughput: 478500888 bytes/sec 456 MB/sec
Writing 500MB of data directly to disk using stdio (fopen,fwrite,fclose) aio (open,aio_write,close) and genio (open,write,close) for 3 iterations yeilded 456 when calling open,write,close consecutively; which is great, it meant I’m on the right track. After a few more tweaks and using genio (open,write,close) I started seeing:
src/writetest –output-dir /data/ –genio –i 3 –mb 500
Beginning cycle write Writing 524288000 bytes, 500 MB
———————————————
gen->open,write,close 0.804808 sec.
———————————————
gen->open,write,close 0.850169 sec.
———————————————
gen->open,write,close 0.867514 sec.
———————————————
gen average write time: 0.840830 sec.
gen average throughput: 623536021 bytes/sec 595 MB/sec
Beginning sequential write Writing 524288000 bytes, 500 MB ———————————————
gen->open,write,close 0.855086 sec.
gen->open,write,close 0.947534 sec.
gen->open,write,close 0.842693 sec.
———————————————
gen average write time: 0.881771 sec.
gen average throughput: 594585215 bytes/sec 567 MB/sec
Beginning cycle write Writing 524288000 bytes, 500 MB
———————————————
gen->open,write,close 0.804808 sec.
———————————————
gen->open,write,close 0.850169 sec.
———————————————
gen->open,write,close 0.867514 sec.
———————————————
gen average write time: 0.840830 sec.
gen average throughput: 623536021 bytes/sec 595 MB/sec
Beginning sequential write Writing 524288000 bytes, 500 MB ———————————————
gen->open,write,close 0.855086 sec.
gen->open,write,close 0.947534 sec.
gen->open,write,close 0.842693 sec.
———————————————
gen average write time: 0.881771 sec.
gen average throughput: 594585215 bytes/sec 567 MB/sec
Perfect, now I getting where I need to go and I know that XFS with the options I specified along with using the generic open,read,write approach would give me the best write times. Feel free to take the above writetest program and use it to tune your filesystem and to make sure that you’re using the fastest possible write method for your chosen filesystem. Be sure to compile with -laio. I thought these findings were worth mentioning.
**************** Update ********************
After changing the OS from a 32-bit to 64-bit system and with a few additional modifications to the kernel, new and even more impressive speeds have been reached.
src/writetest –mb 500 –i 10 –output-dir /data –genio
Beginning cycle write Writing 524288000 bytes, 500 MB
———————————————
gen->open,write,close 0.460466 sec.
———————————————
gen->open,write,close 0.493884 sec.
———————————————
gen->open,write,close 0.494179 sec.
———————————————
gen->open,write,close 0.492141 sec.
———————————————
gen->open,write,close 0.491546 sec.
———————————————
gen->open,write,close 0.490171 sec.
———————————————
gen->open,write,close 0.490388 sec.
———————————————
gen->open,write,close 0.489386 sec.
———————————————
gen->open,write,close 0.489673 sec.
———————————————
gen->open,write,close 0.491476 sec.
———————————————
gen average write time: 0.488331 sec.
gen average throughput: 1073632434 bytes/sec 1024 MB/sec
Beginning sequential write Writing 524288000 bytes, 500 MB ———————————————
gen->open,write,close 0.484176 sec.
gen->open,write,close 0.489009 sec.
gen->open,write,close 0.491489 sec.
gen->open,write,close 0.487558 sec.
gen->open,write,close 0.489555 sec.
gen->open,write,close 0.486413 sec.
gen->open,write,close 0.492973 sec.
gen->open,write,close 0.489842 sec.
gen->open,write,close 0.487276 sec.
gen->open,write,close 0.489895 sec.
———————————————
gen average write time: 0.488819 sec.
gen average throughput: 1072561478 bytes/sec 1023 MB/sec
0 Comments