[Webparser]

- Fixed a issue that Download=1 doesn't work correctly in some cases.
- Added [Measure]-name to log messages.
This commit is contained in:
spx 2010-09-21 14:13:43 +00:00
parent e618ad1777
commit 77ac096f2c

View File

@ -730,10 +730,10 @@ double Update2(UINT id)
} }
else // error else // error
{ {
std::wstring error = L"WebParser: Failed to begin download thread: ["; std::wstring log = L"WebParser: [";
error += urlData->section; log += urlData->section;
error += L"]"; log += L"] Failed to begin download thread.";
Log(error.c_str()); Log(log.c_str());
} }
} }
@ -774,10 +774,10 @@ double Update2(UINT id)
} }
else // error else // error
{ {
std::wstring error = L"WebParser: Failed to begin thread: ["; std::wstring log = L"WebParser: [";
error += urlData->section; log += urlData->section;
error += L"]"; log += L"] Failed to begin thread.";
Log(error.c_str()); Log(log.c_str());
} }
} }
@ -820,9 +820,11 @@ unsigned __stdcall NetworkThreadProc(void* pParam)
} }
else else
{ {
std::wstring error = L"WebParser: Failed to dump debug data: "; std::wstring log = L"WebParser: [";
error += urlData->debugFileLocation; log += urlData->section;
Log(error.c_str()); log += L"] Failed to dump debug data: ";
log += urlData->debugFileLocation;
Log(log.c_str());
} }
} }
@ -894,7 +896,10 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
if (rc == 0) if (rc == 0)
{ {
// The output vector wasn't big enough // The output vector wasn't big enough
Log(L"WebParser: Too many substrings!"); std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] Too many substrings!";
Log(log.c_str());
} }
else else
{ {
@ -904,13 +909,21 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
{ {
for (int i = 0; i < rc; ++i) for (int i = 0; i < rc; ++i)
{ {
WCHAR buffer[512];
const char* substring_start = parseData + ovector[2 * i]; const char* substring_start = parseData + ovector[2 * i];
int substring_length = ovector[2 * i + 1] - ovector[2 * i]; int substring_length = ovector[2 * i + 1] - ovector[2 * i];
substring_length = min(substring_length, 256); substring_length = min(substring_length, 256);
std::string tmpStr(substring_start, substring_length); std::string tmpStr(substring_start, substring_length);
wsprintf(buffer, L"WebParser: (Index %2d) %s", i, ConvertUTF8ToWide(tmpStr.c_str()).c_str());
Log(buffer); WCHAR buffer[32];
wsprintf(buffer, L"%2d", i);
std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] (Index ";
log += buffer;
log += L") ";
log += ConvertUTF8ToWide(tmpStr.c_str());
Log(log.c_str());
} }
} }
@ -924,11 +937,26 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
} }
else else
{ {
Log(L"WebParser: Not enough substrings!"); std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] Not enough substrings!";
Log(log.c_str());
// Clear the old result // Clear the old result
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);
urlData->resultString.clear(); urlData->resultString.clear();
if (urlData->download)
{
if (urlData->downloadFile.empty()) // cache mode
{
if (!urlData->downloadedFile.empty())
{
// Delete old downloaded file
DeleteFile(urlData->downloadedFile.c_str());
}
}
urlData->downloadedFile.clear();
}
LeaveCriticalSection(&g_CriticalSection); LeaveCriticalSection(&g_CriticalSection);
} }
@ -980,10 +1008,10 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
} }
else // error else // error
{ {
std::wstring error = L"WebParser: Failed to begin download thread: ["; std::wstring log = L"WebParser: [";
error += (*i).second->section; log += (*i).second->section;
error += L"]"; log += L"] Failed to begin download thread.";
Log(error.c_str()); Log(log.c_str());
} }
} }
@ -992,11 +1020,26 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
} }
else else
{ {
Log(L"WebParser: Not enough substrings!"); std::wstring log = L"WebParser: [";
log += (*i).second->section;
log += L"] Not enough substrings!";
Log(log.c_str());
// Clear the old result // Clear the old result
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);
((*i).second)->resultString.clear(); ((*i).second)->resultString.clear();
if ((*i).second->download)
{
if ((*i).second->downloadFile.empty()) // cache mode
{
if (!(*i).second->downloadedFile.empty())
{
// Delete old downloaded file
DeleteFile((*i).second->downloadedFile.c_str());
}
}
(*i).second->downloadedFile.clear();
}
LeaveCriticalSection(&g_CriticalSection); LeaveCriticalSection(&g_CriticalSection);
} }
} }
@ -1006,9 +1049,15 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
else else
{ {
// Matching failed: handle error cases // Matching failed: handle error cases
WCHAR buffer[64]; WCHAR buffer[32];
wsprintf(buffer, L"WebParser: Matching error! (%d)\n", rc); wsprintf(buffer, L"%d", rc);
Log(buffer);
std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] Matching error! (";
log += buffer;
log += L")\n";
Log(log.c_str());
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);
urlData->resultString = urlData->errorString; urlData->resultString = urlData->errorString;
@ -1034,9 +1083,17 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
else else
{ {
// Compilation failed: print the error message and exit // Compilation failed: print the error message and exit
WCHAR buffer[512]; WCHAR buffer[32];
wsprintf(buffer, L"WebParser: PCRE compilation failed at offset %d: %s\n", erroffset, ConvertAsciiToWide(error).c_str()); wsprintf(buffer, L"%d", erroffset);
Log(buffer);
std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] PCRE compilation failed at offset ";
log += buffer;
log += L": ";
log += ConvertAsciiToWide(error);
log += L"\n";
Log(log.c_str());
} }
if (urlData->download) if (urlData->download)
@ -1050,10 +1107,10 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
} }
else // error else // error
{ {
std::wstring error = L"WebParser: Failed to begin download thread: ["; std::wstring log = L"WebParser: [";
error += urlData->section; log += urlData->section;
error += L"]"; log += L"] Failed to begin download thread.";
Log(error.c_str()); Log(log.c_str());
} }
} }
else else
@ -1083,12 +1140,17 @@ void ParseData(UrlData* urlData, LPCSTR parseData)
unsigned __stdcall NetworkDownloadThreadProc(void* pParam) unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
{ {
UrlData* urlData = (UrlData*)pParam; UrlData* urlData = (UrlData*)pParam;
const bool download = !urlData->downloadFile.empty();
bool ready = false;
std::wstring url; std::wstring url;
if (urlData->regExp.empty() && urlData->resultString.empty()) if (urlData->regExp.empty() && urlData->resultString.empty())
{ {
url = urlData->url; if (!urlData->url.empty() && urlData->url[0] != L'[')
{
url = urlData->url;
}
} }
else else
{ {
@ -1126,8 +1188,6 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
if (!url.empty()) if (!url.empty())
{ {
bool download = !urlData->downloadFile.empty();
// Create the filename // Create the filename
WCHAR buffer[MAX_PATH] = {0}; WCHAR buffer[MAX_PATH] = {0};
std::wstring fullpath, directory; std::wstring fullpath, directory;
@ -1198,59 +1258,52 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
} }
bool ready = true; ready = true;
if (download) // download mode if (download) // download mode
{ {
std::wstring error; std::wstring log;
if (!PathFileExists(directory.c_str()) || !PathIsDirectory(directory.c_str())) if (!PathFileExists(directory.c_str()) || !PathIsDirectory(directory.c_str()))
{ {
ready = false; ready = false;
error = L"WebParser: Directory not exists: "; log = L"WebParser: [";
error += directory; log += urlData->section;
error += L"\n"; log += L"] Directory not exists: ";
Log(error.c_str()); log += directory;
log += L"\n";
Log(log.c_str());
} }
else if (PathIsDirectory(fullpath.c_str())) else if (PathIsDirectory(fullpath.c_str()))
{ {
ready = false; ready = false;
error = L"WebParser: Path is a directory, not a file: "; log = L"WebParser: [";
error += fullpath; log += urlData->section;
error += L"\n"; log += L"] Path is a directory, not a file: ";
Log(error.c_str()); log += fullpath;
log += L"\n";
Log(log.c_str());
} }
else else if (PathFileExists(fullpath.c_str()))
{ {
DeleteFile(fullpath.c_str()); DWORD attr = GetFileAttributes(fullpath.c_str());
if (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_READONLY))
if (PathFileExists(fullpath.c_str()))
{ {
ready = false; ready = false;
DWORD attr = GetFileAttributes(fullpath.c_str()); log = L"WebParser: [";
if (attr != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_READONLY)) log += urlData->section;
{ log += L"] File is READ-ONLY: ";
error = L"WebParser: File is READ-ONLY: "; log += fullpath;
} log += L"\n";
else Log(log.c_str());
{
error = L"WebParser: Failed to delete file: ";
}
error += fullpath;
error += L"\n";
Log(error.c_str());
} }
} }
} }
else // cache mode else // cache mode
{ {
if (!urlData->downloadedFile.empty())
{
DeleteFile(urlData->downloadedFile.c_str());
}
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);
if (PathFileExists(fullpath.c_str())) if (PathFileExists(fullpath.c_str()))
@ -1269,7 +1322,7 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
// Assign a serial number // Assign a serial number
size_t i = 1; int i = 1;
do do
{ {
wsprintf(buffer, L"_%i", i++); wsprintf(buffer, L"_%i", i++);
@ -1330,12 +1383,14 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
// Write some log info // Write some log info
std::wstring info = L"WebParser: Downloading url "; std::wstring log = L"WebParser: [";
info += url; log += urlData->section;
info += L" to "; log += L"] Downloading url ";
info += fullpath; log += url;
info += L"\n"; log += L" to ";
Log(info.c_str()); log += fullpath;
log += L"\n";
Log(log.c_str());
HRESULT resultCoInitialize = CoInitialize(NULL); // requires before calling URLDownloadToFile function HRESULT resultCoInitialize = CoInitialize(NULL); // requires before calling URLDownloadToFile function
@ -1345,6 +1400,15 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
{ {
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);
if (!download) // cache mode
{
if (!urlData->downloadedFile.empty())
{
// Delete old downloaded file
DeleteFile(urlData->downloadedFile.c_str());
}
}
// Convert LFN to 8.3 filename if the path contains blank character // Convert LFN to 8.3 filename if the path contains blank character
if (fullpath.find_first_of(L' ') != std::wstring::npos) if (fullpath.find_first_of(L' ') != std::wstring::npos)
{ {
@ -1377,6 +1441,8 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
else else
{ {
ready = false;
if (!download) // cache mode if (!download) // cache mode
{ {
// Delete empty file // Delete empty file
@ -1384,11 +1450,14 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
wsprintf(buffer, L"result=0x%08X, COM=0x%08X", result, resultCoInitialize); wsprintf(buffer, L"result=0x%08X, COM=0x%08X", result, resultCoInitialize);
std::wstring error = L"WebParser: Download failed (";
error += buffer; std::wstring log = L"WebParser: [";
error += L"): "; log += urlData->section;
error += url; log += L"] Download failed (";
Log(error.c_str()); log += buffer;
log += L"): ";
log += url;
Log(log.c_str());
} }
if (SUCCEEDED(resultCoInitialize)) if (SUCCEEDED(resultCoInitialize))
@ -1398,14 +1467,38 @@ unsigned __stdcall NetworkDownloadThreadProc(void* pParam)
} }
else else
{ {
std::wstring error = L"WebParser: Download failed: "; std::wstring log = L"WebParser: [";
error += url; log += urlData->section;
Log(error.c_str()); log += L"] Download failed: ";
log += url;
Log(log.c_str());
} }
} }
else else
{ {
Log(L"WebParser: The url is empty.\n"); std::wstring log = L"WebParser: [";
log += urlData->section;
log += L"] The url is empty.\n";
Log(log.c_str());
}
if (!ready) // download failed
{
EnterCriticalSection(&g_CriticalSection);
if (!download) // cache mode
{
if (!urlData->downloadedFile.empty())
{
// Delete old downloaded file
DeleteFile(urlData->downloadedFile.c_str());
}
}
// Clear old downloaded filename
urlData->downloadedFile.clear();
LeaveCriticalSection(&g_CriticalSection);
} }
EnterCriticalSection(&g_CriticalSection); EnterCriticalSection(&g_CriticalSection);