src/db.cpp
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2012 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "db.h"
#include "net.h"
#include "hash.h"
#include "main.h"
#include "ui_interface.h"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#ifndef WIN32
#include "sys/stat.h"
#endif
using namespace std;
unsigned int nWalletDBUpdated;
//
// CDB
//
CDBEnv bitdb;
void CDBEnv::EnvShutdown()
{
if (!fDbEnvInit)
return;
fDbEnvInit = false;
int ret = dbenv.close(0);
if (ret != 0)
LogPrintf("EnvShutdown exception: %s (%d)\n", DbEnv::strerror(ret), ret);
if (!fMockDb)
DbEnv(0).remove(strPath.c_str(), 0);
}
CDBEnv::CDBEnv() : dbenv(DB_CXX_NO_EXCEPTIONS)
{
fDbEnvInit = false;
fMockDb = false;
}
CDBEnv::~CDBEnv()
{
EnvShutdown();
}
void CDBEnv::Close()
{
EnvShutdown();
}
bool CDBEnv::Open(boost::filesystem::path pathEnv_)
{
if (fDbEnvInit)
return true;
boost::this_thread::interruption_point();
pathEnv = pathEnv_;
boost::filesystem::path pathDataDir = pathEnv;
strPath = pathDataDir.string();
boost::filesystem::path pathLogDir = pathDataDir / "database";
boost::filesystem::create_directory(pathLogDir);
boost::filesystem::path pathErrorFile = pathDataDir / "db.log";
LogPrintf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
unsigned int nEnvFlags = 0;
if (GetBoolArg("-privdb", true))
nEnvFlags |= DB_PRIVATE;
int nDbCache = GetArg("-dbcache", 25);
dbenv.set_lg_dir(pathLogDir.string().c_str());
dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
dbenv.set_lg_bsize(1048576);
dbenv.set_lg_max(10485760);
// Bugfix: Bump lk_max_locks default to 537000, to safely handle reorgs with up to 5 blocks reversed
// dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_locks(537000);
dbenv.set_lk_max_objects(10000);
dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
dbenv.set_flags(DB_AUTO_COMMIT, 1);
dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
#ifdef DB_LOG_AUTO_REMOVE
dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
#endif
int ret = dbenv.open(strPath.c_str(),
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_RECOVER |
nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret != 0)
return error("CDB() : error %s (%d) opening database environment", DbEnv::strerror(ret), ret);
fDbEnvInit = true;
fMockDb = false;
return true;
}
void CDBEnv::MakeMock()
{
if (fDbEnvInit)
throw runtime_error("CDBEnv::MakeMock(): already initialized");
boost::this_thread::interruption_point();
LogPrintf("CDBEnv::MakeMock()\n");
dbenv.set_cachesize(1, 0, 1);
dbenv.set_lg_bsize(10485760*4);
dbenv.set_lg_max(10485760);
dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_objects(10000);
dbenv.set_flags(DB_AUTO_COMMIT, 1);
#ifdef DB_LOG_IN_MEMORY
dbenv.log_set_config(DB_LOG_IN_MEMORY, 1);
#endif
int ret = dbenv.open(NULL,
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_PRIVATE,
S_IRUSR | S_IWUSR);
if (ret > 0)
throw runtime_error(strprintf("CDBEnv::MakeMock(): error %d opening database environment", ret));
fDbEnvInit = true;
fMockDb = true;
}
CDBEnv::VerifyResult CDBEnv::Verify(std::string strFile, bool (*recoverFunc)(CDBEnv& dbenv, std::string strFile))
{
LOCK(cs_db);
assert(mapFileUseCount.count(strFile) == 0);
Db db(&dbenv, 0);
int result = db.verify(strFile.c_str(), NULL, NULL, 0);
if (result == 0)
return VERIFY_OK;
else
if (recoverFunc == NULL)
return RECOVER_FAIL;
// Try to recover:
bool fRecovered = (*recoverFunc)(*this, strFile);
return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
}
bool CDBEnv::Salvage(std::string strFile, bool fAggressive,
std::vector<CDBEnv::KeyValPair >& vResult)
{
LOCK(cs_db);
assert(mapFileUseCount.count(strFile) == 0);
u_int32_t flags = DB_SALVAGE;
if (fAggressive) flags |= DB_AGGRESSIVE;
stringstream strDump;
Db db(&dbenv, 0);
int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
if (result == DB_VERIFY_BAD)
{
LogPrintf("Error: Salvage found errors, all data may not be recoverable.\n");
if (!fAggressive)
{
LogPrintf("Error: Rerun with aggressive mode to ignore errors and continue.\n");
return false;
}
}
if (result != 0 && result != DB_VERIFY_BAD)
{
LogPrintf("ERROR: db salvage failed: %d\n",result);
return false;
}
// Format of bdb dump is ascii lines:
// header lines...
// HEADER=END
// hexadecimal key
// hexadecimal value
// ... repeated
// DATA=END
string strLine;
while (!strDump.eof() && strLine != "HEADER=END")
getline(strDump, strLine); // Skip past header
std::string keyHex, valueHex;
while (!strDump.eof() && keyHex != "DATA=END")
{
getline(strDump, keyHex);
if (keyHex != "DATA=END")
{
getline(strDump, valueHex);
vResult.push_back(make_pair(ParseHex(keyHex),ParseHex(valueHex)));
}
}
return (result == 0);
}
void CDBEnv::CheckpointLSN(const std::string& strFile)
{
dbenv.txn_checkpoint(0, 0, 0);
if (fMockDb)
return;
dbenv.lsn_reset(strFile.c_str(), 0);
}
CDB::CDB(const std::string& strFilename, const char* pszMode) :
pdb(NULL), activeTxn(NULL)
{
int ret;
fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
if (strFilename.empty())
return;
bool fCreate = strchr(pszMode, 'c');
unsigned int nFlags = DB_READ_UNCOMMITTED | DB_THREAD; // get must be called with DB_READ_UNCOMMITTED also for it to apply
if (fCreate)
nFlags |= DB_CREATE;
{
LOCK(bitdb.cs_db);
if (!bitdb.Open(GetDataDir()))
throw runtime_error("env open failed");
strFile = strFilename;
++bitdb.mapFileUseCount[strFile];
pdb = bitdb.mapDb[strFile];
if (pdb == NULL)
{
pdb = new Db(&bitdb.dbenv, 0);
bool fMockDb = bitdb.IsMock();
if (fMockDb)
{
DbMpoolFile*mpf = pdb->get_mpf();
ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
if (ret != 0)
throw runtime_error(strprintf("CDB : Failed to configure for no temp file backing for database %s", strFile.c_str()));
}
ret = pdb->open(NULL, // Txn pointer
fMockDb ? NULL : strFile.c_str(), // Filename
fMockDb ? strFile.c_str() : "main", // Logical db name
DB_BTREE, // Database type
nFlags, // Flags
0);
if (ret != 0)
{
delete pdb;
pdb = NULL;
--bitdb.mapFileUseCount[strFile];
strFile = "";
throw runtime_error(strprintf("CDB : Error %d, can't open database %s", ret, strFile.c_str()));
}
if (fCreate && !Exists(string("version")))
{
bool fTmp = fReadOnly;
fReadOnly = false;
WriteVersion(CLIENT_VERSION);
fReadOnly = fTmp;
}
bitdb.mapDb[strFile] = pdb;
}
}
}
static bool IsChainFile(std::string strFile)
{
if (strFile == "blkindex.dat")
return true;
return false;
}
void CDB::Close()
{
if (!pdb)
return;
if (activeTxn)
activeTxn->abort();
activeTxn = NULL;
pdb = NULL;
// Flush database activity from memory pool to disk log
unsigned int nMinutes = 0;
if (fReadOnly)
nMinutes = 1;
if (IsChainFile(strFile))
nMinutes = 2;
if (IsChainFile(strFile) && IsInitialBlockDownload())
nMinutes = 5;
bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0);
{
LOCK(bitdb.cs_db);
--bitdb.mapFileUseCount[strFile];
}
}
void CDBEnv::CloseDb(const string& strFile)
{
{
LOCK(cs_db);
if (mapDb[strFile] != NULL)
{
// Close the database handle
Db* pdb = mapDb[strFile];
pdb->close(0);
delete pdb;
mapDb[strFile] = NULL;
}
}
}
bool CDBEnv::RemoveDb(const string& strFile)
{
this->CloseDb(strFile);
LOCK(cs_db);
int rc = dbenv.dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT);
return (rc == 0);
}
bool CDB::Rewrite(const string& strFile, const char* pszSkip)
{
for (;;)
{
boost::this_thread::interruption_point();
{
LOCK(bitdb.cs_db);
if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0)
{
// Flush log data to the dat file
bitdb.CloseDb(strFile);
bitdb.CheckpointLSN(strFile);
bitdb.mapFileUseCount.erase(strFile);
bool fSuccess = true;
LogPrintf("Rewriting %s...\n", strFile.c_str());
string strFileRes = strFile + ".rewrite";
{ // surround usage of db with extra {}
CDB db(strFile.c_str(), "r");
Db* pdbCopy = new Db(&bitdb.dbenv, 0);
int ret = pdbCopy->open(NULL, // Txn pointer
strFileRes.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
DB_CREATE, // Flags
0);
if (ret > 0)
{
LogPrintf("Cannot create database file %s\n", strFileRes.c_str());
fSuccess = false;
}
Dbc* pcursor = db.GetCursor();
if (pcursor)
while (fSuccess)
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT);
if (ret == DB_NOTFOUND)
{
pcursor->close();
break;
}
else if (ret != 0)
{
pcursor->close();
fSuccess = false;
break;
}
if (pszSkip &&
strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
continue;
if (strncmp(&ssKey[0], "\x07version", 8) == 0)
{
// Update version:
ssValue.clear();
ssValue << CLIENT_VERSION;
}
Dbt datKey(&ssKey[0], ssKey.size());
Dbt datValue(&ssValue[0], ssValue.size());
int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
if (ret2 > 0)
fSuccess = false;
}
if (fSuccess)
{
db.Close();
bitdb.CloseDb(strFile);
if (pdbCopy->close(0))
fSuccess = false;
delete pdbCopy;
}
}
if (fSuccess)
{
Db dbA(&bitdb.dbenv, 0);
if (dbA.remove(strFile.c_str(), NULL, 0))
fSuccess = false;
Db dbB(&bitdb.dbenv, 0);
if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
fSuccess = false;
}
if (!fSuccess)
LogPrintf("Rewriting of %s FAILED!\n", strFileRes.c_str());
return fSuccess;
}
}
MilliSleep(100);
}
return false;
}
void CDBEnv::Flush(bool fShutdown)
{
int64_t nStart = GetTimeMillis();
// Flush log data to the actual data file
// on all files that are not in use
LogPrintf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started");
if (!fDbEnvInit)
return;
{
LOCK(cs_db);
map<string, int>::iterator mi = mapFileUseCount.begin();
while (mi != mapFileUseCount.end())
{
string strFile = (*mi).first;
int nRefCount = (*mi).second;
LogPrintf("%s refcount=%d\n", strFile.c_str(), nRefCount);
if (nRefCount == 0)
{
// Move log data to the dat file
CloseDb(strFile);
LogPrintf("%s checkpoint\n", strFile.c_str());
dbenv.txn_checkpoint(0, 0, 0);
if (!IsChainFile(strFile) || fDetachDB) {
LogPrintf("%s detach\n", strFile.c_str());
if (!fMockDb)
dbenv.lsn_reset(strFile.c_str(), 0);
}
LogPrintf("%s closed\n", strFile.c_str());
mapFileUseCount.erase(mi++);
}
else
mi++;
}
LogPrintf("DBFlush(%s)%s ended %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started", GetTimeMillis() - nStart);
if (fShutdown)
{
char** listp;
if (mapFileUseCount.empty())
{
dbenv.log_archive(&listp, DB_ARCH_REMOVE);
Close();
}
}
}
}
//
// CAddrDB
//
CAddrDB::CAddrDB()
{
pathAddr = GetDataDir() / "peers.dat";
}
bool CAddrDB::Write(const CAddrMan& addr)
{
// Generate random temporary filename
unsigned short randv = 0;
RAND_bytes((unsigned char *)&randv, sizeof(randv));
std::string tmpfn = strprintf("peers.dat.%04x", randv);
// serialize addresses, checksum data up to that point, then append csum
CDataStream ssPeers(SER_DISK, CLIENT_VERSION);
ssPeers << FLATDATA(Params().MessageStart());
ssPeers << addr;
uint256 hash = Hash(ssPeers.begin(), ssPeers.end());
ssPeers << hash;
// open temp output file, and associate with CAutoFile
boost::filesystem::path pathTmp = GetDataDir() / tmpfn;
FILE *file = fopen(pathTmp.string().c_str(), "wb");
CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION);
if (!fileout)
return error("CAddrman::Write() : open failed");
// Write and commit header, data
try {
fileout << ssPeers;
}
catch (std::exception &e) {
return error("CAddrman::Write() : I/O error");
}
FileCommit(fileout);
fileout.fclose();
// replace existing peers.dat, if any, with new peers.dat.XXXX
if (!RenameOver(pathTmp, pathAddr))
return error("CAddrman::Write() : Rename-into-place failed");
return true;
}
bool CAddrDB::Read(CAddrMan& addr)
{
// open input file, and associate with CAutoFile
FILE *file = fopen(pathAddr.string().c_str(), "rb");
CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION);
if (!filein)
return error("CAddrman::Read() : open failed");
// use file size to size memory buffer
int fileSize = boost::filesystem::file_size(pathAddr);
int dataSize = fileSize - sizeof(uint256);
// Don't try to resize to a negative number if file is small
if ( dataSize < 0 ) dataSize = 0;
vector<unsigned char> vchData;
vchData.resize(dataSize);
uint256 hashIn;
// read data and checksum from file
try {
filein.read((char *)&vchData[0], dataSize);
filein >> hashIn;
}
catch (std::exception &e) {
return error("CAddrman::Read() 2 : I/O error or stream data corrupted");
}
filein.fclose();
CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION);
// verify stored checksum matches input data
uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end());
if (hashIn != hashTmp)
return error("CAddrman::Read() : checksum mismatch; data corrupted");
unsigned char pchMsgTmp[4];
try {
// de-serialize file header (pchMessageStart magic number) and
ssPeers >> FLATDATA(pchMsgTmp);
// verify the network matches ours
if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp)))
return error("CAddrman::Read() : invalid network magic number");
// de-serialize address data into one CAddrMan object
ssPeers >> addr;
}
catch (std::exception &e) {
return error("CAddrman::Read() : I/O error or stream data corrupted");
}
return true;
}