Core/DataStores

* Changed loading dbc/db2. Worldserver no longer accepts dbc and db2 files directly inside '/dbc/*/ folder but expects them to be structured exactly as mapextractor creates them 'dbc/locale/*'
* Fixed loading locale db2 files if they were not placed in root dbc directory
Tools/Mapextractor: Fixed extracting locale dbcs

It is recommended to re-extract dbc files by running 'mapextractor -e 2' instructing the tool to only create dbc/db2 files skipping maps.

src/server/game/DataStores/DB2Stores.cpp

@@ -134,7 +134,7 @@ typedef std::list<std::string> DB2StoreProblemList;
 uint32 DB2FilesCount = 0;
 
 template<class T>
-inline void LoadDB2(uint32& availableDb2Locales, DB2StoreProblemList& errlist, DB2Manager::StorageMap& stores, DB2Storage<T>* storage, std::string const& db2_path)
+inline void LoadDB2(uint32& availableDb2Locales, DB2StoreProblemList& errlist, DB2Manager::StorageMap& stores, DB2Storage<T>* storage, std::string const& db2Path, uint32 defaultLocale)
 {
     // compatibility format and C++ structure sizes
     ASSERT(DB2FileLoader::GetFormatRecordSize(storage->GetFormat()) == sizeof(T),
@@ -143,17 +143,17 @@ inline void LoadDB2(uint32& availableDb2Locales, DB2StoreProblemList& errlist, D
 
     ++DB2FilesCount;
 
-    if (storage->Load(db2_path, uint32(sWorld->GetDefaultDbcLocale())))
+    if (storage->Load(db2Path + localeNames[defaultLocale] + '/', defaultLocale))
     {
         storage->LoadFromDB();
 
         for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
         {
-            if (uint32(sWorld->GetDefaultDbcLocale()) == i)
+            if (defaultLocale == i)
                 continue;
 
             if (availableDb2Locales & (1 << i))
-                if (!storage->LoadStringsFrom((db2_path + localeNames[i] + '/'), i))
+                if (!storage->LoadStringsFrom((db2Path + localeNames[i] + '/'), i))
                     availableDb2Locales &= ~(1 << i);             // mark as not available for speedup next checks
 
             storage->LoadStringsFromDB(i);
@@ -162,7 +162,7 @@ inline void LoadDB2(uint32& availableDb2Locales, DB2StoreProblemList& errlist, D
     else
     {
         // sort problematic db2 to (1) non compatible and (2) nonexistent
-        if (FILE* f = fopen((db2_path + storage->GetFileName()).c_str(), "rb"))
+        if (FILE* f = fopen((db2Path + storage->GetFileName()).c_str(), "rb"))
         {
             std::ostringstream stream;
             stream << storage->GetFileName() << " exists, and has " << storage->GetFieldCount() << " field(s) (expected " << strlen(storage->GetFormat())
@@ -178,16 +178,16 @@ inline void LoadDB2(uint32& availableDb2Locales, DB2StoreProblemList& errlist, D
     stores[storage->GetHash()] = storage;
 }
 
-void DB2Manager::LoadStores(std::string const& dataPath)
+void DB2Manager::LoadStores(std::string const& dataPath, uint32 defaultLocale)
 {
     uint32 oldMSTime = getMSTime();
 
-    std::string db2Path = GetDBCLocaleFolder(dataPath);
+    std::string db2Path = dataPath + "dbc/";
 
     DB2StoreProblemList bad_db2_files;
     uint32 availableDb2Locales = 0xFF;
 
-#define LOAD_DB2(store) LoadDB2(availableDb2Locales, bad_db2_files, _stores, &store, db2Path)
+#define LOAD_DB2(store) LoadDB2(availableDb2Locales, bad_db2_files, _stores, &store, db2Path, defaultLocale)
 
     LOAD_DB2(sAreaGroupMemberStore);
     LOAD_DB2(sAreaGroupStore);
@@ -501,7 +501,7 @@ void DB2Manager::LoadStores(std::string const& dataPath)
     // error checks
     if (bad_db2_files.size() >= DB2FilesCount)
     {
-        TC_LOG_ERROR("misc", "\nIncorrect DataDir value in worldserver.conf or ALL required *.db2 files (%d) not found by path: %sdb2", DB2FilesCount, dataPath.c_str());
+        TC_LOG_ERROR("misc", "\nIncorrect DataDir value in worldserver.conf or ALL required *.db2 files (%d) not found by path: %sdbc/%s/", DB2FilesCount, dataPath.c_str(), localeNames[defaultLocale]);
         exit(1);
     }
     else if (!bad_db2_files.empty())

src/server/game/DataStores/DB2Stores.h

@@ -157,7 +157,7 @@ public:
         return instance;
     }
 
-    void LoadStores(std::string const& dataPath);
+    void LoadStores(std::string const& dataPath, uint32 defaultLocale);
     DB2StorageBase const* GetStorage(uint32 type) const;
 
     void LoadHotfixData();

src/server/game/DataStores/DBCStores.cpp

@@ -23,9 +23,7 @@
 #include "DBCfmt.h"
 #include "Timer.h"
 #include "DB2Stores.h"
-#include "Config.h"
 
-#include <boost/filesystem.hpp>
 #include <map>
 
 typedef std::map<uint16, uint32> AreaFlagByAreaID;
@@ -168,7 +166,6 @@ SpellCategoryStore sSpellsByCategoryStore;
 PetFamilySpellsStore sPetFamilySpellsStore;
 SpellEffectScallingByEffectId sSpellEffectScallingByEffectId;
 
-
 DBCStorage <SpellScalingEntry> sSpellScalingStore(SpellScalingEntryfmt);
 DBCStorage <SpellTargetRestrictionsEntry> sSpellTargetRestrictionsStore(SpellTargetRestrictionsEntryfmt);
 DBCStorage <SpellLevelsEntry> sSpellLevelsStore(SpellLevelsEntryfmt);
@@ -200,7 +197,7 @@ uint32 DBCFileCount = 0;
 uint32 GameTableCount = 0;
 
 template<class T>
-inline void LoadDBC(uint32& availableDbcLocales, StoreProblemList& errors, DBCStorage<T>& storage, std::string const& dbcPath, std::string const& filename, std::string const* customFormat = NULL, std::string const* customIndexName = NULL)
+inline void LoadDBC(uint32& availableDbcLocales, StoreProblemList& errors, DBCStorage<T>& storage, std::string const& dbcPath, std::string const& filename, uint32 defaultLocale, std::string const* customFormat = NULL, std::string const* customIndexName = NULL)
 {
     // compatibility format and C++ structure sizes
     ASSERT(DBCFileLoader::GetFormatRecordSize(storage.GetFormat()) == sizeof(T),
@@ -208,7 +205,7 @@ inline void LoadDBC(uint32& availableDbcLocales, StoreProblemList& errors, DBCSt
         filename.c_str(), DBCFileLoader::GetFormatRecordSize(storage.GetFormat()), uint32(sizeof(T)));
 
     ++DBCFileCount;
-    std::string dbcFilename = dbcPath + filename;
+    std::string dbcFilename = dbcPath + localeNames[defaultLocale] + '/' + filename;
     SqlDbc * sql = NULL;
     if (customFormat)
         sql = new SqlDbc(&filename, customFormat, customIndexName, storage.GetFormat());
@@ -300,16 +297,16 @@ inline void LoadGameTable(StoreProblemList& errors, std::string const& tableName
     }
 }
 
-void LoadDBCStores(const std::string& dataPath)
+void LoadDBCStores(const std::string& dataPath, uint32 defaultLocale)
 {
     uint32 oldMSTime = getMSTime();
 
-    std::string dbcPath = GetDBCLocaleFolder(dataPath);
+    std::string dbcPath = dataPath + "dbc/";
 
     StoreProblemList bad_dbc_files;
     uint32 availableDbcLocales = 0xFFFFFFFF;
 
-#define LOAD_DBC(store, file) LoadDBC(availableDbcLocales, bad_dbc_files, store, dbcPath, file)
+#define LOAD_DBC(store, file) LoadDBC(availableDbcLocales, bad_dbc_files, store, dbcPath, file, defaultLocale)
 
     LOAD_DBC(sAchievementStore, "Achievement.dbc"/*, &CustomAchievementfmt, &CustomAchievementIndex*/);//20201
     LOAD_DBC(sAnimKitStore, "AnimKit.dbc");//20201
@@ -503,7 +500,7 @@ void LoadDBCStores(const std::string& dataPath)
     // error checks
     if (bad_dbc_files.size() >= DBCFileCount)
     {
-        TC_LOG_ERROR("misc", "Incorrect DataDir value in worldserver.conf or ALL required *.dbc files (%d) not found by path: %sdbc", DBCFileCount, dataPath.c_str());
+        TC_LOG_ERROR("misc", "Incorrect DataDir value in worldserver.conf or ALL required *.dbc files (%d) not found by path: %sdbc/%s/", DBCFileCount, dataPath.c_str(), localeNames[defaultLocale]);
         exit(1);
     }
     else if (!bad_dbc_files.empty())
@@ -530,11 +527,11 @@ void LoadDBCStores(const std::string& dataPath)
     TC_LOG_INFO("server.loading", ">> Initialized %d DBC data stores in %u ms", DBCFileCount, GetMSTimeDiffToNow(oldMSTime));
 }
 
-void LoadGameTables(const std::string& dataPath)
+void LoadGameTables(const std::string& dataPath, uint32 defaultLocale)
 {
     uint32 oldMSTime = getMSTime();
 
-    std::string dbcPath = GetDBCLocaleFolder(dataPath);
+    std::string dbcPath = dataPath + "dbc/" + localeNames[defaultLocale] + '/';
 
     StoreProblemList bad_dbc_files;
 
@@ -566,7 +563,7 @@ void LoadGameTables(const std::string& dataPath)
     // error checks
     if (bad_dbc_files.size() >= GameTableCount)
     {
-        TC_LOG_ERROR("misc", "Incorrect DataDir value in worldserver.conf or ALL required *.dbc GameTable files (%d) not found by path: %sdbc", DBCFileCount, dataPath.c_str());
+        TC_LOG_ERROR("misc", "Incorrect DataDir value in worldserver.conf or ALL required *.dbc GameTable files (%d) not found by path: %sdbc/%s/", DBCFileCount, dataPath.c_str(), localeNames[defaultLocale]);
         exit(1);
     }
     else if (!bad_dbc_files.empty())
@@ -920,21 +917,3 @@ SkillRaceClassInfoEntry const* GetSkillRaceClassInfo(uint32 skill, uint8 race, u
 
     return NULL;
 }
-
-std::string GetDBCLocaleFolder(std::string const& dataPath)
-{
-    using namespace boost::filesystem;
-
-    const std::string dbcPath = dataPath + "dbc/";
-    directory_iterator dir_iter(dbcPath);
-    directory_iterator end_iter;
-
-    const int locale = sConfigMgr->GetIntDefault("DBC.Locale", LOCALE_enUS);
-    for(; dir_iter != end_iter; ++dir_iter)
-        if (is_directory(*dir_iter) && GetLocaleByName((*dir_iter).path().filename().string()) == locale)
-            // Return the full path appending detected locale folder
-            return (*dir_iter).path().string() + "/";
-
-    // Empty folder, or it has files but not subfolders (probably the DBC files).
-    return dbcPath;
-}

src/server/game/DataStores/DBCStores.h

@@ -226,8 +226,7 @@ extern DBCStorage <WMOAreaTableEntry>            sWMOAreaTableStore;
 //extern DBCStorage <WorldMapAreaEntry>           sWorldMapAreaStore; -- use Zone2MapCoordinates and Map2ZoneCoordinates
 extern DBCStorage <WorldSafeLocsEntry>           sWorldSafeLocsStore;
 
-void LoadDBCStores(const std::string& dataPath);
-void LoadGameTables(const std::string& dataPath);
-std::string GetDBCLocaleFolder(std::string const& dataPath);
+void LoadDBCStores(const std::string& dataPath, uint32 defaultLocale);
+void LoadGameTables(const std::string& dataPath, uint32 defaultLocale);
 
 #endif

src/server/game/Maps/TransportMgr.h

@@ -99,7 +99,7 @@ typedef std::map<uint32, TransportAnimation> TransportAnimationContainer;
 
 class TransportMgr
 {
-        friend void DB2Manager::LoadStores(std::string const&);
+        friend void DB2Manager::LoadStores(std::string const&, uint32);
 
     public:
         static TransportMgr* instance()

src/server/game/World/World.cpp

@@ -419,7 +419,7 @@ void World::LoadConfigSettings(bool reload)
 
     m_defaultDbcLocale = LocaleConstant(sConfigMgr->GetIntDefault("DBC.Locale", 0));
 
-    if (m_defaultDbcLocale >= TOTAL_LOCALES)
+    if (m_defaultDbcLocale >= TOTAL_LOCALES || m_defaultDbcLocale < LOCALE_enUS)
     {
         TC_LOG_ERROR("server.loading", "Incorrect DBC.Locale! Must be >= 0 and < %d (set to 0)", TOTAL_LOCALES);
         m_defaultDbcLocale = LOCALE_enUS;
@@ -1459,15 +1459,15 @@ void World::SetInitialWorldSettings()
 
     TC_LOG_INFO("server.loading", "Initialize data stores...");
     ///- Load DBCs
-    LoadDBCStores(m_dataPath);
+    LoadDBCStores(m_dataPath, m_defaultDbcLocale);
     ///- Load DB2s
-    sDB2Manager.LoadStores(m_dataPath);
+    sDB2Manager.LoadStores(m_dataPath, m_defaultDbcLocale);
     TC_LOG_INFO("misc", "Loading hotfix info...");
     sDB2Manager.LoadHotfixData();
     ///- Close hotfix database - it is only used during DB2 loading
     HotfixDatabase.Close();
     ///- Load GameTables
-    LoadGameTables(m_dataPath);
+    LoadGameTables(m_dataPath, m_defaultDbcLocale);
 
     sSpellMgr->LoadPetFamilySpellsStore();
 

src/tools/map_extractor/CMakeLists.txt

@@ -12,6 +12,7 @@
 file(GLOB_RECURSE sources *.cpp *.h)
 
 include_directories (
+  ${CMAKE_SOURCE_DIR}/src/common
   ${CMAKE_SOURCE_DIR}/src/server/shared
   ${CMAKE_SOURCE_DIR}/dep/CascLib/src
   ${CMAKE_CURRENT_SOURCE_DIR}
@@ -26,6 +27,7 @@ add_executable(mapextractor
 
 target_link_libraries(mapextractor
   casc
+  common
   ${BZIP2_LIBRARIES}
   ${ZLIB_LIBRARIES}
   ${Boost_LIBRARIES}

src/tools/map_extractor/System.cpp

@@ -36,6 +36,10 @@
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
 
+#include "Common.h"
+#ifdef PLATFORM_WINDOWS
+#undef PLATFORM_WINDOWS
+#endif
 #include "DBFilesClientList.h"
 #include "CascLib.h"
 #include "dbcfile.h"
@@ -98,8 +102,8 @@ map_id *map_ids;
 uint16 *areas;
 uint16 *LiqType;
 #define MAX_PATH_LENGTH 128
-char output_path[MAX_PATH_LENGTH] = ".";
-char input_path[MAX_PATH_LENGTH] = ".";
+char output_path[MAX_PATH_LENGTH];
+char input_path[MAX_PATH_LENGTH];
 uint32 maxAreaId = 0;
 
 // **************************************************
@@ -127,9 +131,9 @@ float CONF_flat_liquid_delta_limit = 0.001f; // If max - min less this value - l
 
 uint32 CONF_Locale = 0;
 
-#define LOCALES_COUNT 17
+#define CASC_LOCALES_COUNT 17
 
-char const* Locales[LOCALES_COUNT] =
+char const* CascLocaleNames[CASC_LOCALES_COUNT] =
 {
     "none", "enUS",
     "koKR", "unknown",
@@ -142,12 +146,28 @@ char const* Locales[LOCALES_COUNT] =
     "ptPT"
 };
 
+uint32 WowLocaleToCascLocaleFlags[12] =
+{
+    CASC_LOCALE_ENUS | CASC_LOCALE_ENGB,
+    CASC_LOCALE_KOKR,
+    CASC_LOCALE_FRFR,
+    CASC_LOCALE_DEDE,
+    CASC_LOCALE_ZHCN,
+    CASC_LOCALE_ZHTW,
+    CASC_LOCALE_ESES,
+    CASC_LOCALE_ESMX,
+    CASC_LOCALE_RURU,
+    0,
+    CASC_LOCALE_PTBR | CASC_LOCALE_PTPT,
+    CASC_LOCALE_ITIT,
+};
+
 void CreateDir(std::string const& path)
 {
     if (chdir(path.c_str()) == 0)
     {
-            chdir("../");
-            return;
+        chdir("../");
+        return;
     }
 
 #ifdef _WIN32
@@ -178,6 +198,7 @@ void Usage(char const* prg)
         "-o set output path (max %d characters)\n"\
         "-e extract only MAP(1)/DBC(2) - standard: both(3)\n"\
         "-f height stored as int (less map size but lost some accuracy) 1 by default\n"\
+        "-l dbc locale\n"\
         "Example: %s -f 0 -i \"c:\\games\\game\"\n", prg, MAX_PATH_LENGTH - 1, MAX_PATH_LENGTH - 1, prg);
     exit(1);
 }
@@ -234,8 +255,8 @@ void HandleArgs(int argc, char* arg[])
             case 'l':
                 if (c + 1 < argc)                            // all ok
                 {
-                    for (uint32 i = 0; i < LOCALES_COUNT; ++i)
-                        if (!strcmp(arg[c + 1], Locales[i]))
+                    for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
+                        if (!strcmp(arg[c + 1], localeNames[i]))
                             CONF_Locale = 1 << i;
                     ++c;
                 }
@@ -254,13 +275,13 @@ void HandleArgs(int argc, char* arg[])
 uint32 ReadBuild(int locale)
 {
     // include build info file also
-    std::string filename  = std::string("component.wow-") + Locales[locale] + ".txt";
+    std::string filename  = std::string("component.wow-") + localeNames[locale] + ".txt";
     //printf("Read %s file... ", filename.c_str());
 
     HANDLE dbcFile;
     if (!CascOpenFile(CascStorage, filename.c_str(), CASC_LOCALE_ALL, 0, &dbcFile))
     {
-        printf("Locale %s not installed.\n", Locales[locale]);
+        printf("Locale %s not installed.\n", localeNames[locale]);
         return 0;
     }
 
@@ -1124,7 +1145,7 @@ void ExtractMaps(uint32 build)
 bool ExtractFile(HANDLE fileInArchive, char const* filename)
 {
     FILE* output = fopen(filename, "wb");
-    if(!output)
+    if (!output)
     {
         printf("Can't create the output file '%s'\n", filename);
         return false;
@@ -1152,10 +1173,12 @@ void ExtractDBFilesClient(int l)
     outputPath += "/dbc/";
 
     CreateDir(outputPath);
-    outputPath += Locales[l];
+    outputPath += localeNames[l];
     outputPath += "/";
     CreateDir(outputPath);
 
+    printf("locale %s output path %s\n", localeNames[l], outputPath.c_str());
+
     uint32 index = 0;
     uint32 count = 0;
     char const* fileName = DBFilesClientList[index];
@@ -1163,8 +1186,8 @@ void ExtractDBFilesClient(int l)
     while (fileName)
     {
         std::string filename = fileName;
-        if (CascOpenFile(CascStorage, (filename = (filename + ".db2")).c_str(), 1 << l, 0, &dbcFile) ||
-            CascOpenFile(CascStorage, (filename = (filename.substr(0, filename.length() - 4) + ".dbc")).c_str(), 1 << l, 0, &dbcFile))
+        if (CascOpenFile(CascStorage, (filename = (filename + ".db2")).c_str(), WowLocaleToCascLocaleFlags[l], 0, &dbcFile) ||
+            CascOpenFile(CascStorage, (filename = (filename.substr(0, filename.length() - 4) + ".dbc")).c_str(), WowLocaleToCascLocaleFlags[l], 0, &dbcFile))
         {
             filename = outputPath + filename.substr(filename.rfind('\\') + 1);
 
@@ -1175,7 +1198,7 @@ void ExtractDBFilesClient(int l)
             CascCloseFile(dbcFile);
         }
         else
-            printf("Unable to open file %s in the archive for locale %s: %s\n", fileName, Locales[l], HumanReadableCASCError(GetLastError()));
+            printf("Unable to open file %s in the archive for locale %s: %s\n", fileName, localeNames[l], HumanReadableCASCError(GetLastError()));
 
         fileName = DBFilesClientList[++index];
     }
@@ -1183,17 +1206,17 @@ void ExtractDBFilesClient(int l)
     printf("Extracted %u files\n\n", count);
 }
 
-bool OpenCascStorage()
+bool OpenCascStorage(int locale)
 {
     try
     {
         boost::filesystem::path const storage_dir(boost::filesystem::canonical(input_path) / "Data");
-        if (!CascOpenStorage(storage_dir.string().c_str(), 0, &CascStorage))
+        if (!CascOpenStorage(storage_dir.string().c_str(), WowLocaleToCascLocaleFlags[locale], &CascStorage))
         {
-            printf("error opening casc storage '%s': %s\n", storage_dir.string().c_str(), HumanReadableCASCError(GetLastError()));
+            printf("error opening casc storage '%s' locale %s: %s\n", storage_dir.string().c_str(), localeNames[locale], HumanReadableCASCError(GetLastError()));
             return false;
         }
-        printf("opened casc storage '%s'\n", storage_dir.string().c_str());
+        printf("opened casc storage '%s' locale %s\n", storage_dir.string().c_str(), localeNames[locale]);
         return true;
     }
     catch (boost::filesystem::filesystem_error& error)
@@ -1208,27 +1231,35 @@ int main(int argc, char * arg[])
     printf("Map & DBC Extractor\n");
     printf("===================\n");
 
+    boost::filesystem::path current(boost::filesystem::current_path());
+    strcpy(input_path, current.string().c_str());
+    strcpy(output_path, current.string().c_str());
+
     HandleArgs(argc, arg);
 
     int FirstLocale = -1;
     uint32 build = 0;
 
-    if (!OpenCascStorage())
-    {
-        return 1;
-    }
-
-    for (int i = 0; i < LOCALES_COUNT; ++i)
+    for (int i = 0; i < TOTAL_LOCALES; ++i)
     {
         if (CONF_Locale && !(CONF_Locale & (1 << i)))
             continue;
 
+        if (i == LOCALE_none)
+            continue;
+
+        if (!OpenCascStorage(i))
+            continue;
+
         if ((CONF_extract & EXTRACT_DBC) == 0)
         {
             FirstLocale = i;
             build = ReadBuild(i);
             if (!build)
+            {
+                CascCloseStorage(CascStorage);
                 continue;
+            }
 
             printf("Detected client build: %u\n\n", build);
             break;
@@ -1237,10 +1268,14 @@ int main(int argc, char * arg[])
         //Extract DBC files
         uint32 tempBuild = ReadBuild(i);
         if (!tempBuild)
+        {
+            CascCloseStorage(CascStorage);
             continue;
+        }
 
-        printf("Detected client build %u for locale %s\n\n", tempBuild, Locales[i]);
+        printf("Detected client build %u for locale %s\n\n", tempBuild, localeNames[i]);
         ExtractDBFilesClient(i);
+        CascCloseStorage(CascStorage);
 
         if (FirstLocale < 0)
         {
@@ -1257,11 +1292,10 @@ int main(int argc, char * arg[])
 
     if (CONF_extract & EXTRACT_MAP)
     {
-        printf("Using locale: %s\n", Locales[FirstLocale]);
-
+        OpenCascStorage(0);
         ExtractMaps(build);
+        CascCloseStorage(CascStorage);
     }
 
-    CascCloseStorage(CascStorage);
     return 0;
 }