用Windows的文件映射机制, 实现大批量数据的快速存储(转载)
//const FILE_CACHE_SIZE = $40000000; // = 1GB
//const FILE_CACHE_SIZE = $01E00000; // = 30MB
const FILE_CACHE_SIZE = 100*64*1024; // = 6.4MB
TFileCache = class(TObject)
private
mMappingViewSize: INT64;
mWriteMappingOffset: INT64;
mWriteBufferOffset: INT64;
mReadMappingOffset: INT64;
FFileHandle: integer;
FMappingHandle: integer;
mWriteBuffer: pointer;
mReadBuffer: pointer;
public
{ Public declarations }
constructor Create;
destructor Destroy; override;
function CreateFileCache(const AFileName:String): boolean;
// 向镜像文件中追加数据。
function AppendData(const pData; pDataLength: integer): boolean;
// 从镜像文件的指定位置读取数据。
function ReadData(pAddressOffset: INT64; var pData; pDataLength: integer): boolean;
end;
constructor TFileCache.Create;
begin
inherited Create;
mMappingViewSize := 100 * (64*1024); //window's default block size is 64KB
FFileHandle := INVALID_HANDLE_VALUE;
FMappingHandle := 0;
mWriteMappingOffset := 0;
mWriteBuffer := nil;
mWriteBufferOffset := 0;
mReadMappingOffset := 0;
mReadBuffer := nil;
end;
function TFileCache.CreateFileCache(const AFileName: String): boolean;
begin
FFileHandle := CreateFile(PChar(AFileName), GENERIC_READ or GENERIC_WRITE, FILE_SHARE_READ, nil, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
if FFileHandle = INVALID_HANDLE_VALUE then
begin
raise Exception.Create('Error when open file');
end;
FMappingHandle := CreateFileMapping(FFileHandle, nil, PAGE_READWRITE, 0, DWORD(FILE_CACHE_SIZE and $FFFFFFFF), nil); //DWORD(FILE_CACHE_SIZE shr 32)
if FMappingHandle=0 then begin
raise Exception.Create('Error when mapping file');
end;
mWriteMappingOffset := 0;
mWriteBuffer := nil;
mWriteBufferOffset := 0;
mReadMappingOffset := 0;
mReadBuffer := nil;
Result := true;
end;
destructor TFileCache.Destroy;
begin
if (mWriteBuffer <> nil) then
UnmapViewOfFile(mWriteBuffer);
if (mReadBuffer <> nil) then
UnmapViewOfFile(mReadBuffer);
if (FMappingHandle <> 0) then
CloseHandle(FMappingHandle);
if (FFileHandle <> INVALID_HANDLE_VALUE) then
CloseHandle(FFileHandle);
inherited Destroy;
end;
function TFileCache.AppendData(const pData; pDataLength: integer): boolean;
var
datawrote: integer;
datacanwrite: integer;
datatowrite: integer;
actualdatatowrite: integer;
begin
datawrote := 0;
while (datawrote < pDataLength) do
begin
datacanwrite := mMappingViewSize - mWriteBufferOffset;
if (mWriteBuffer<>nil) and (datacanwrite <= 0) then
begin
UnmapViewOfFile(mWriteBuffer);
mWriteBuffer := nil;
mWriteMappingOffset := mWriteMappingOffset + mMappingViewSize;
end;
if (mWriteBuffer = nil) then
begin
mWriteBuffer := MapViewOfFile(FMappingHandle, FILE_MAP_WRITE, DWORD(mWriteMappingOffset shr 32), DWORD(mWriteMappingOffset and $FFFFFFFF), mMappingViewSize);
mWriteBufferOffset := 0;
datacanwrite := mMappingViewSize - mWriteBufferOffset;
if (mWriteBuffer = nil) then
begin
raise Exception.Create('Error when map view of file.');
end;
end;
datatowrite := pDataLength - datawrote;
if (datacanwrite >= datatowrite) then
actualdatatowrite := datatowrite
else
actualdatatowrite := datacanwrite;
CopyMemory(Pointer(Longint(mWriteBuffer) + mWriteBufferOffset), Pointer(Longint(Pointer(pData)) + datawrote), actualdatatowrite);
// memcpy((PBYTE)mWriteBuffer+mWriteBufferOffset, (PBYTE)pData+datawrote, actualdatatowrite);
mWriteBufferOffset := mWriteBufferOffset + actualdatatowrite;
datawrote := datawrote + actualdatatowrite;
end;
end;
function TFileCache.ReadData(pAddressOffset: INT64; var pData; pDataLength: integer): boolean;
var
datareaded: integer;
datacanread: integer;
datatoread: integer;
actualdatatoread: integer;
begin
datareaded := 0;
while (datareaded < pDataLength) do
begin
datacanread := mReadMappingOffset + mMappingViewSize - pAddressOffset - datareaded;
if (mReadBuffer<>nil) and ((datacanread <= 0) or (datacanread > mMappingViewSize)) then
begin
UnmapViewOfFile(mReadBuffer);
mReadBuffer := nil;
end;
if (mReadBuffer = nil) then
begin
mReadMappingOffset := (pAddressOffset + datareaded) div mMappingViewSize * mMappingViewSize;
mReadBuffer := MapViewOfFile(FMappingHandle, FILE_MAP_READ, DWORD(mReadMappingOffset shr 32), DWORD(mReadMappingOffset and $FFFFFFFF), mMappingViewSize); //DWORD(mReadMappingOffset shr 32)
datacanread := mReadMappingOffset + mMappingViewSize - pAddressOffset - datareaded;
if (mReadBuffer = nil) then
begin
raise Exception.Create('Error when map view of file.');
// return false;
end;
end;
datatoread := pDataLength - datareaded;
if (datacanread >= datatoread) then
actualdatatoread := datatoread
else
actualdatatoread := datacanread;
//Pointer(Longint(pData) + datareaded)
CopyMemory(Pointer(Longint(@pData) + datareaded), Pointer(LongInt(mReadBuffer) + pAddressOffset - mReadMappingOffset + datareaded), actualdatatoread);
// memcpy((PBYTE)pData+datareaded, (PBYTE)mReadBuffer+pAddressOffset-mReadMappingOffset+datareaded, actualdatatoread);
datareaded := datareaded + actualdatatoread;
end;
Result := true;
end;
//使用例子
//读取
procedure TForm1.Button2Click(Sender: TObject);
var
FC: TFileCache;
s: string;
begin
FC := TFileCache.Create;
FC.CreateFileCache('1.txt');
SetString(S, nil, 100);
FC.ReadData(0, Pointer(S)^, 100);
Memo1.Lines.Add(S);
FC.Free;
end;
//写入
procedure TForm1.Button4Click(Sender: TObject);
var
FC: TFileCache;
s1: string;
begin
FC := TFileCache.Create;
FC.CreateFileCache('1.txt');
s1 := '你好,哈哈';
FC.AppendData(Pointer(s1), Length(s1));
FC.AppendData(Pointer(s1), Length(s1));
FC.AppendData(Pointer(s1), Length(s1));
FC.Free;
end;