发新话题
打印

[SDK/API/系统] 用Windows的文件映射机制, 实现大批量数据的快速存储(转载)

用Windows的文件映射机制, 实现大批量数据的快速存储(转载)

//const FILE_CACHE_SIZE = $40000000; // = 1GB
//const FILE_CACHE_SIZE = $01E00000; // = 30MB
const FILE_CACHE_SIZE = 100*64*1024; // = 6.4MB

TFileCache = class(TObject)
private
mMappingViewSize: INT64;
mWriteMappingOffset: INT64;
mWriteBufferOffset: INT64;
mReadMappingOffset: INT64;

FFileHandle: integer;
FMappingHandle: integer;

mWriteBuffer: pointer;
mReadBuffer: pointer;
public
{ Public declarations }
constructor Create;
destructor Destroy; override;

function CreateFileCache(const AFileName:String): boolean;

// 向镜像文件中追加数据。
function AppendData(const pData; pDataLength: integer): boolean;

// 从镜像文件的指定位置读取数据。
function ReadData(pAddressOffset: INT64; var pData; pDataLength: integer): boolean;

end;

constructor TFileCache.Create;
begin
inherited Create;

mMappingViewSize := 100 * (64*1024); //window's default block size is 64KB

FFileHandle := INVALID_HANDLE_VALUE;
FMappingHandle := 0;

mWriteMappingOffset := 0;
mWriteBuffer := nil;
mWriteBufferOffset := 0;

mReadMappingOffset := 0;
mReadBuffer := nil;

end;

function TFileCache.CreateFileCache(const AFileName: String): boolean;
begin

FFileHandle := CreateFile(PChar(AFileName), GENERIC_READ or GENERIC_WRITE, FILE_SHARE_READ, nil, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
if FFileHandle = INVALID_HANDLE_VALUE then
begin
raise Exception.Create('Error when open file');
end;

FMappingHandle := CreateFileMapping(FFileHandle, nil, PAGE_READWRITE, 0, DWORD(FILE_CACHE_SIZE and $FFFFFFFF), nil); //DWORD(FILE_CACHE_SIZE shr 32)
if FMappingHandle=0 then begin
raise Exception.Create('Error when mapping file');
end;

mWriteMappingOffset := 0;
mWriteBuffer := nil;
mWriteBufferOffset := 0;
mReadMappingOffset := 0;
mReadBuffer := nil;
Result := true;
end;

destructor TFileCache.Destroy;
begin
if (mWriteBuffer <> nil) then
UnmapViewOfFile(mWriteBuffer);

if (mReadBuffer <> nil) then
UnmapViewOfFile(mReadBuffer);

if (FMappingHandle <> 0) then
CloseHandle(FMappingHandle);

if (FFileHandle <> INVALID_HANDLE_VALUE) then
CloseHandle(FFileHandle);

inherited Destroy;
end;

function TFileCache.AppendData(const pData; pDataLength: integer): boolean;
var
datawrote: integer;
datacanwrite: integer;
datatowrite: integer;
actualdatatowrite: integer;
begin
datawrote := 0;

while (datawrote < pDataLength) do
begin

datacanwrite := mMappingViewSize - mWriteBufferOffset;
if (mWriteBuffer<>nil) and (datacanwrite <= 0) then
begin
UnmapViewOfFile(mWriteBuffer);
mWriteBuffer := nil;
mWriteMappingOffset := mWriteMappingOffset + mMappingViewSize;
end;

if (mWriteBuffer = nil) then
begin
mWriteBuffer := MapViewOfFile(FMappingHandle, FILE_MAP_WRITE, DWORD(mWriteMappingOffset shr 32), DWORD(mWriteMappingOffset and $FFFFFFFF), mMappingViewSize);
mWriteBufferOffset := 0;
datacanwrite := mMappingViewSize - mWriteBufferOffset;

if (mWriteBuffer = nil) then
begin
raise Exception.Create('Error when map view of file.');
end;
end;

datatowrite := pDataLength - datawrote;
if (datacanwrite >= datatowrite) then
actualdatatowrite := datatowrite
else
actualdatatowrite := datacanwrite;

CopyMemory(Pointer(Longint(mWriteBuffer) + mWriteBufferOffset), Pointer(Longint(Pointer(pData)) + datawrote), actualdatatowrite);
// memcpy((PBYTE)mWriteBuffer+mWriteBufferOffset, (PBYTE)pData+datawrote, actualdatatowrite);
mWriteBufferOffset := mWriteBufferOffset + actualdatatowrite;
datawrote := datawrote + actualdatatowrite;
end;
end;

function TFileCache.ReadData(pAddressOffset: INT64; var pData; pDataLength: integer): boolean;
var
datareaded: integer;
datacanread: integer;
datatoread: integer;
actualdatatoread: integer;
begin

datareaded := 0;

while (datareaded < pDataLength) do
begin

datacanread := mReadMappingOffset + mMappingViewSize - pAddressOffset - datareaded;

if (mReadBuffer<>nil) and ((datacanread <= 0) or (datacanread > mMappingViewSize)) then
begin
UnmapViewOfFile(mReadBuffer);
mReadBuffer := nil;
end;

if (mReadBuffer = nil) then
begin
mReadMappingOffset := (pAddressOffset + datareaded) div mMappingViewSize * mMappingViewSize;

mReadBuffer := MapViewOfFile(FMappingHandle, FILE_MAP_READ, DWORD(mReadMappingOffset shr 32), DWORD(mReadMappingOffset and $FFFFFFFF), mMappingViewSize); //DWORD(mReadMappingOffset shr 32)
datacanread := mReadMappingOffset + mMappingViewSize - pAddressOffset - datareaded;

if (mReadBuffer = nil) then
begin
raise Exception.Create('Error when map view of file.');
// return false;
end;
end;

datatoread := pDataLength - datareaded;
if (datacanread >= datatoread) then
actualdatatoread := datatoread
else
actualdatatoread := datacanread;

//Pointer(Longint(pData) + datareaded)

CopyMemory(Pointer(Longint(@pData) + datareaded), Pointer(LongInt(mReadBuffer) + pAddressOffset - mReadMappingOffset + datareaded), actualdatatoread);
// memcpy((PBYTE)pData+datareaded, (PBYTE)mReadBuffer+pAddressOffset-mReadMappingOffset+datareaded, actualdatatoread);

datareaded := datareaded + actualdatatoread;
end;
Result := true;
end;

//使用例子

//读取
procedure TForm1.Button2Click(Sender: TObject);
var
FC: TFileCache;

s: string;

begin
FC := TFileCache.Create;

FC.CreateFileCache('1.txt');

SetString(S, nil, 100);
FC.ReadData(0, Pointer(S)^, 100);
Memo1.Lines.Add(S);

FC.Free;

end;

//写入
procedure TForm1.Button4Click(Sender: TObject);
var
FC: TFileCache;

s1: string;

begin
FC := TFileCache.Create;

FC.CreateFileCache('1.txt');

s1 := '你好,哈哈';

FC.AppendData(Pointer(s1), Length(s1));

FC.AppendData(Pointer(s1), Length(s1));

FC.AppendData(Pointer(s1), Length(s1));


FC.Free;

end;
妖城欢迎您!

TOP

发新话题