bpo-31523: Reliability improvements to the Windows build files (#3900)

8 years ago · 2084b30e54
5 changed files with 56 additions and 27 deletions
--- a/PCbuild/pyproject.props
+++ b/PCbuild/pyproject.props
@ -6,7 +6,7 @@
    <OutDir>$(BuildPath)</OutDir>
    <OutDir Condition="!HasTrailingSlash($(OutDir))">$(OutDir)\</OutDir>
    <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir>
-    <IntDir>$(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir>
+    <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir>
    <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName>
    <TargetName>$(TargetName)$(PyDebugExt)</TargetName>
    <GenerateManifest>false</GenerateManifest>
--- a/Tools/msi/msi.props
+++ b/Tools/msi/msi.props
@ -48,7 +48,7 @@
    </ItemGroup>

    <PropertyGroup>
-        <IntermediateOutputPath>$(Py_IntDir)\$(Configuration)_$(Platform)_Setup\$(OutputName)</IntermediateOutputPath>
+        <IntermediateOutputPath>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\msi_$(OutputName)</IntermediateOutputPath>
        <IntermediateOutputPath Condition="'$(OutputSuffix)' != ''">$(IntermediateOutputPath)_$(OutputSuffix)</IntermediateOutputPath>
        <OutputPath Condition="'$(OutputPath)' == ''">$(BuildPath)</OutputPath>
        <OutputPath Condition="!HasTrailingSlash($(OutputPath))">$(OutputPath)\</OutputPath>
--- a/Tools/msi/purge.py
+++ b/Tools/msi/purge.py
@ -25,7 +25,7 @@ except LookupError:
    sys.exit(1)

 URL = "https://www.python.org/ftp/python/{}/".format(m.group(1))
-
+REL = m.group(2) or ''

 FILES = [
    "core.msi",
@ -64,7 +64,7 @@ PATHS = [
    "python-{}-webinstall.exe".format(m.group(0)),
    "python-{}-amd64.exe".format(m.group(0)),
    "python-{}-amd64-webinstall.exe".format(m.group(0)),
-] + ["win32{}/{}".format(m.group(2), f) for f in FILES] + ["amd64{}/{}".format(m.group(2), f) for f in FILES]
+] + ["win32{}/{}".format(REL, f) for f in FILES] + ["amd64{}/{}".format(REL, f) for f in FILES]

 print('Purged:')
 for n in PATHS:
--- a/Tools/msi/uploadrelease.bat
+++ b/Tools/msi/uploadrelease.bat
@ -8,8 +8,9 @@ set HOST=
 set USER=
 set TARGET=
 set DRYRUN=false
+set NOUPLOAD=
 set NOGPG=
-set PURGE_OPTION=/p:Purge=true
+set NOPURGE=
 set NOTEST=

 :CheckOpts
@ -21,8 +22,9 @@ if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts
 if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts
 if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts
 if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts
+if "%1" EQU "--skip-upload" (set NOUPLOAD=true) && shift && goto CheckOpts
 if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts
-if "%1" EQU "--skip-purge" (set PURGE_OPTION=) && shift && goto CheckOpts
+if "%1" EQU "--skip-purge" (set NOPURGE=true) && shift && goto CheckOpts
 if "%1" EQU "--skip-test" (set NOTEST=true) && shift && goto CheckOpts
 if "%1" EQU "-T" (set NOTEST=true) && shift && goto CheckOpts
 if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1
@ -52,17 +54,43 @@ if defined NOGPG (
 call "%PCBUILD%find_msbuild.bat" %MSBUILD%
 if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 pushd "%D%"
-%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION%
-%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION%
+if not defined NOUPLOAD (
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86
+    if errorlevel 1 goto :failed
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false
+    if errorlevel 1 goto :failed
+)
+if not defined NOPURGE (
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge
+)
 if not defined NOTEST (
-    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86
-    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64
+    call :test x86
+    if errorlevel 1 goto :failed
+    call :test x64
+    if errorlevel 1 goto :failed
 )
 %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86
+if errorlevel 1 goto :failed
 %MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false
+if errorlevel 1 goto :failed
+
 popd
 exit /B 0

+:test
+%MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1
+if errorlevel 1 (
+    echo Test failed - purging and retrying
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge
+    if errorlevel 1 exit /B
+    %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1
+)
+exit /B
+
+:failed
+popd
+exit /B
+
 :Help
 echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h]
 echo.
--- a/Tools/msi/uploadrelease.proj
+++ b/Tools/msi/uploadrelease.proj
@ -10,7 +10,6 @@
        <IncludeDoc Condition="'$(IncludeDoc)' == ''">true</IncludeDoc>
        <BuildForRelease Condition="'$(BuildForRelease)' == ''">true</BuildForRelease>
        <DryRun Condition="'$(DryRun)' == ''">false</DryRun>
-        <Purge Condition="'$(Purge)' == ''">false</Purge>
    </PropertyGroup>

    <Import Project="msi.props" />
@ -43,7 +42,8 @@
    <Target Name="_RunGpg" Condition="'$(GPG)' != ''" Inputs="@(File)" Outputs="$(IntermediateOutputPath)\gpg\%(FileName)%(Extension).asc">
        <MakeDir Directories="$(IntermediateOutputPath)gpg" />
        <Delete Files="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc" Condition="Exists('$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc')" />
-        <Exec Command="&quot;$(GPG)&quot; -ba -o &quot;$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc&quot; &quot;%(File.FullPath)&quot;" />
+        <Exec Command="&quot;$(GPG)&quot; -ba -o &quot;$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc&quot; &quot;%(File.FullPath)&quot;"
+              IgnoreExitCode="false" />
        <ItemGroup>
            <File Include="$(IntermediateOutputPath)\gpg\%(File.FileName)%(File.Extension).asc">
                <CopyTo>%(File.CopyTo)</CopyTo>
@ -52,23 +52,15 @@
    </Target>
    
    <Target Name="_Upload" Condition="!$(DryRun)">
-        <Exec Command="&quot;$(PLINK)&quot; $(User)@$(Host) mkdir %(File.CopyTo) ^&amp;^&amp; chgrp downloads %(File.CopyTo) ^&amp;^&amp; chmod g-w,o+rx %(File.CopyTo)
-&quot;$(PSCP)&quot; @(File,' ') $(User)@$(Host):%(File.CopyTo)
-&quot;$(PLINK)&quot; $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*
-" />
+        <Exec Command="&quot;$(PLINK)&quot; $(User)@$(Host) mkdir %(File.CopyTo) ^&amp;^&amp; chgrp downloads %(File.CopyTo) ^&amp;^&amp; chmod g-w,o+rx %(File.CopyTo)" />
+        <Exec Command="&quot;$(PSCP)&quot; @(File,' ') $(User)@$(Host):%(File.CopyTo)" />
+        <Exec Command="&quot;$(PLINK)&quot; $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" />
    </Target>
    
    <Target Name="_PrintNames" Condition="$(DryRun)">
-        <Exec Command="echo &quot;$(PLINK)&quot; $(User)@$(Host) mkdir %(File.CopyTo) ^&amp;^&amp; chgrp downloads %(File.CopyTo) ^&amp;^&amp; chmod g-w,o+rx %(File.CopyTo)
-echo &quot;$(PSCP)&quot; @(File,' ') $(User)@$(Host):%(File.CopyTo)
-echo &quot;$(PLINK)&quot; $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*
-echo.
-echo." />
-    </Target>
-    
-    <Target Name="_Purge" Condition="$(Purge) and !$(DryRun)">
-        <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" />
-        <Exec Command="&quot;$(PythonExe)&quot; purge.py $(PythonVersion)" />
+        <Exec Command="echo &quot;$(PLINK)&quot; $(User)@$(Host) mkdir %(File.CopyTo) ^&amp;^&amp; chgrp downloads %(File.CopyTo) ^&amp;^&amp; chmod g-w,o+rx %(File.CopyTo)" />
+        <Exec Command="echo &quot;$(PSCP)&quot; @(File,' ') $(User)@$(Host):%(File.CopyTo)" />
+        <Exec Command="echo &quot;$(PLINK)&quot; $(User)@$(Host) chgrp downloads %(File.CopyTo)/*; chmod g-w,o+r %(File.CopyTo)/*" />
    </Target>
    
    <Target Name="_TestLayout">
@ -82,6 +74,7 @@ echo." />
                <LogFile>$(OutputPath)\%(Filename)_layoutlog\%(Filename).log</LogFile>
            </WebInstaller>
        </ItemGroup>
+        <Error Text="Could not find installer" Condition="@(WebInstaller) == ''" />
        <RemoveDir Directories="%(WebInstaller.SourceDir)" Condition="Exists('%(WebInstaller.SourceDir)')" />
        <RemoveDir Directories="%(WebInstaller.LayoutDir)" Condition="Exists('%(WebInstaller.LayoutDir)')" />
        <RemoveDir Directories="%(WebInstaller.LogDir)" Condition="Exists('%(WebInstaller.LogDir)')" />
@ -95,14 +88,22 @@ echo." />
        <Message Text="Successfully downloaded %(WebInstaller.Filename)%(WebInstaller.Extension) layout" Importance="high" />
    </Target>
    
-    <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload;_Purge" />
+    <Target Name="Upload" DependsOnTargets="_ValidateProperties;_RunGpg;_PrintNames;_Upload" />
    <Target Name="Test" DependsOnTargets="_TestLayout" />
    
+    <Target Name="Purge">
+        <Error Condition="!Exists('$(PythonExe)')" Text="No Python executable available at $(PythonExe)" />
+        <Exec Command="echo &quot;$(PythonExe)&quot; purge.py $(PythonVersion)" Condition="$(DryRun)" />
+        <Exec Command="&quot;$(PythonExe)&quot; purge.py $(PythonVersion)" Condition="!$(DryRun)" />
+        <Message Text="Purged uploaded files" Importance="high" />
+    </Target>
+    
    <Target Name="ShowHashes">
        <ItemGroup>
            <UserFiles Include="@(File)" Condition="'%(File.CopyTo)' == '$(EXETarget)'" />
        </ItemGroup>
        
+        <Error Text="No files generated" Condition="@(UserFiles) == ''" />
        <Exec Command="&quot;$(PythonExe)&quot; generate_md5.py @(UserFiles->'&quot;%(FullPath)&quot;',' ')" />
    </Target>