deploy release and daily build pdbs (#2680)

* [WIP] deploy release and daily build pdbs to s3

* First pass at windows symbol generation for #2680

* Kill the conflicted compiler flag warning

* Tweaks to windows compiler flags for release builds, more qt-ish release symbols and ltcg

* Deploy tagged release symbols and CI builds to different S3 folders

* Support LTCG on iOS

* Just turn off ltcg for iOS if it is going to be such a pain

.appveyor.yml

@@ -30,6 +30,14 @@ build_script:
   - mkdir %SHADOW_BUILD_DIR% && cd %SHADOW_BUILD_DIR% && C:\Qt\5.5\msvc2013\bin\qmake -r CONFIG-=debug_and_release CONFIG+=%CONFIG% CONFIG+=WarningsAsErrorsOn %APPVEYOR_BUILD_FOLDER%\qgroundcontrol.pro
   - cd %SHADOW_BUILD_DIR% && jom
   - if "%CONFIG%" EQU "installer" ( copy %SHADOW_BUILD_DIR%\release\qgroundcontrol-installer.exe %APPVEYOR_BUILD_FOLDER%\qgroundcontrol-installer.exe )
+# Generate the source server information to embed in the PDB
+  - '"C:\Program Files (x86)\Windows Kits\8.1\Debuggers\x64\srcsrv\srctool.exe" -r -u "%SHADOW_BUILD_DIR%\release\qgroundcontrol.pdb" | grep qgroundcontrol | grep -v moc_ | grep -v libs\\mavlink | grep -v build_windows_install | python %APPVEYOR_BUILD_FOLDER%\deploy\genPDBsrcsrv.py > %SHADOW_BUILD_DIR%\release\qgroundcontrol.pdb.srcsrv'
+# write the source server info
+  - '"C:\Program Files (x86)\Windows Kits\8.1\Debuggers\x64\srcsrv\pdbstr.exe" -w -i:%SHADOW_BUILD_DIR%\release\qgroundcontrol.pdb.srcsrv -p:%SHADOW_BUILD_DIR%\release\qgroundcontrol.pdb -s:srcsrv'
+# build the symbol / PE store
+  - 'cd %APPVEYOR_BUILD_FOLDER% && "C:\Program Files (x86)\Windows Kits\8.1\Debuggers\x64\symstore.exe" add /compress /s symbols /f %SHADOW_BUILD_DIR%\release\qgroundcontrol.pdb /t qgroundcontrol'
+  - 'cd %APPVEYOR_BUILD_FOLDER% && "C:\Program Files (x86)\Windows Kits\8.1\Debuggers\x64\symstore.exe" add /compress /s symbols /f %SHADOW_BUILD_DIR%\release\qgroundcontrol.exe /t qgroundcontrol'
+  - ps: 'cd $env:appveyor_build_folder; .\deploy\lowercaseify_symbolstore.ps1'
 
 test_script:
   - if "%CONFIG%" EQU "debug" ( %SHADOW_BUILD_DIR%\debug\qgroundcontrol --unittest )
@@ -38,6 +46,9 @@ artifacts:
   - path: qgroundcontrol-installer.exe
     name: qgcinstaller
 
+  - path: symbols\**\*.*_
+    name: symbols
+
 deploy:
   - provider: S3
     access_key_id:
@@ -50,6 +61,30 @@ deploy:
     artifact: qgcinstaller
     on:
       CONFIG: installer
+  - provider: S3
+    access_key_id:
+      secure: IGAojLMqokL+76DbdulmWDA3MTsxEBBi3ReVVSqTy9c=
+    secret_access_key:
+      secure: RiYqaR+3T2PMNz2j5ur8LCA6H/Zfd4jTX33CZE5iBxm+zaz4QLs25p0B7prpaoNN
+    bucket: qgroundcontrol
+    set_public: true
+    folder: cisyms
+    artifact: symbols
+    on:
+      CONFIG: installer
+      appveyor_repo_tag: false
+  - provider: S3
+    access_key_id:
+      secure: IGAojLMqokL+76DbdulmWDA3MTsxEBBi3ReVVSqTy9c=
+    secret_access_key:
+      secure: RiYqaR+3T2PMNz2j5ur8LCA6H/Zfd4jTX33CZE5iBxm+zaz4QLs25p0B7prpaoNN
+    bucket: qgroundcontrol
+    set_public: true
+    folder: releasesyms
+    artifact: symbols
+    on:
+      CONFIG: installer
+      appveyor_repo_tag: true
   - provider: GitHub
     auth_token:
       secure: dzWLqd0eTKjrAWC5LgqVnwLemmrNdddGA2ZZn/OthAP37mwCLkP2C1zil7ivmEE8

QGCCommon.pri

@@ -178,11 +178,15 @@ MacBuild | LinuxBuild {
 }
 
 WindowsBuild {
+    QMAKE_CFLAGS_RELEASE -= -Zc:strictStrings
+    QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO -= -Zc:strictStrings
+    QMAKE_CXXFLAGS_RELEASE -= -Zc:strictStrings
+    QMAKE_CXXFLAGS_RELEASE_WITH_DEBUGINFO -= -Zc:strictStrings
     QMAKE_CXXFLAGS_WARN_ON += /W3 \
         /wd4996 \   # silence warnings about deprecated strcpy and whatnot
         /wd4005 \   # silence warnings about macro redefinition
-        /wd4290 \   # ignore exception specifications
+        /wd4290     # ignore exception specifications
-        /Zc:strictStrings-  # work around win 8.1 sdk sapi.h problem
+
     WarningsAsErrorsOn {
         QMAKE_CXXFLAGS_WARN_ON += /WX
     }
@@ -194,14 +198,21 @@ WindowsBuild {
 
 ReleaseBuild {
     DEFINES += QT_NO_DEBUG
-    WindowsBuild {
+    CONFIG += force_debug_info  # Enable debugging symbols on release builds
-        # Use link time code generation for better optimization (I believe this is supported in MSVC Express, but not 100% sure)
+    !iOSBuild {
-        QMAKE_LFLAGS_LTCG = /LTCG
+        CONFIG += ltcg              # Turn on link time code generation
-        QMAKE_CFLAGS_LTCG = -GL
+    }
 
-        # Turn on debugging information so we can collect good crash dumps from release builds
+    WindowsBuild {
-        QMAKE_CXXFLAGS_RELEASE += /Zi
+        # Enable function level linking and enhanced optimized debugging
-        QMAKE_LFLAGS_RELEASE += /DEBUG
+        QMAKE_CFLAGS_RELEASE   += /Gy /Zo
+        QMAKE_CXXFLAGS_RELEASE += /Gy /Zo
+        QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO   += /Gy /Zo
+        QMAKE_CXXFLAGS_RELEASE_WITH_DEBUGINFO += /Gy /Zo
+
+        # Eliminate duplicate COMDATs
+        QMAKE_LFLAGS_RELEASE += /OPT:ICF
+        QMAKE_LFLAGS_RELEASE_WITH_DEBUGINFO += /OPT:ICF
     }
 }
 

deploy/genPDBsrcsrv.py

@@ -0,0 +1,31 @@
+import fileinput
+import sys
+import os
+import glob
+
+def get_actual_filename(name):
+    dirs = name.split('\\')
+    # disk letter
+    test_name = [dirs[0].upper()]
+    for d in dirs[1:]:
+        test_name += ["%s[%s]" % (d[:-1], d[-1])]
+    res = glob.glob('\\'.join(test_name))
+    if not res:
+        #File not found
+        return None
+    return res[0]
+    
+filelist = [get_actual_filename(x.rstrip()) for x in fileinput.input(['-'])]
+prefix_len = len(os.path.commonprefix(filelist))
+
+print("""SRCSRV: ini ------------------------------------------------
+VERSION=2
+SRCSRV: variables ------------------------------------------
+SRCSRVVERCTRL=https
+SRCSRVTRG=https://raw.github.com/mavlink/qgroundcontrol/%s/%%var2%%
+SRCSRV: source files ---------------------------------------""" % os.environ['APPVEYOR_REPO_COMMIT'])
+for line in filelist:
+    if line is not None:
+        print('%s*%s' % (line, line[prefix_len:].replace('\\','/')))
+        
+print("SRCSRV: end ------------------------------------------------")

deploy/lowercaseify_symbolstore.ps1

@@ -0,0 +1,14 @@
+Add-Type -TypeDefinition @'
+    using System;
+    using System.Runtime.InteropServices;
+
+    public class NativeMethods
+    {
+        [DllImport("kernel32.dll", EntryPoint="MoveFileW", SetLastError=true,
+                   CharSet=CharSet.Unicode, ExactSpelling=true,
+                   CallingConvention=CallingConvention.StdCall)]
+        public static extern bool MoveFile(string lpExistingFileName, string lpNewFileName);
+    }
+'@
+
+Get-ChildItem ".\symbols" -recurse | ForEach-Object {[NativeMethods]::MoveFile($_.FullName,[io.path]::combine((Split-Path $_.FullName -Parent),$_.Name.ToLower()))}

deploy/qgroundcontrol_installer.nsi

@@ -67,7 +67,7 @@ Section
 
 doinstall:
   SetOutPath $INSTDIR
-  File /r build_windows_install\release\*.*
+  File /r /x qgroundcontrol.pdb /x qgroundcontrol.lib /x qgroundcontrol.exp build_windows_install\release\*.* 
   File deploy\px4driver.msi
   WriteUninstaller $INSTDIR\QGroundControl_uninstall.exe
   WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\QGroundControl" "DisplayName" "QGroundControl"