File _service:tar_scm:0016-Add-BiSheng-Autotuner-sup... of Package clang

File _service:tar_scm:0016-Add-BiSheng-Autotuner-support-for-LLVM-compiler.patch of Package clang (Revision a7e57019a9e799cbb3b26b9bf8e89fb3)

Currently displaying revision a7e57019a9e799cbb3b26b9bf8e89fb3 , Show latest

From a9863e2b6e6783aa9be0b9d1d187084fd4b32a3a Mon Sep 17 00:00:00 2001
From: Muhammad Asif Manzoor <muhammad.asif.manzoor1@huawei.com>
Date: Thu, 21 Mar 2024 12:50:38 -0400
Subject: Add BiSheng Autotuner support for LLVM compiler

Automatic tuning is an automatic iterative process that optimizes a given
program by manipulating compilation options for optimal performance.
BiSheng Autotuner provides a resumable interface for tuning process. BiSheng
Autotuner can tune 1) individual code segments/blocks (fine grain turning) like
loops, callsites, instructions, etc. and 2) entire modules/programs (coarse
grain tuning) for compiler flags, pass ordering, etc.
This patch enables LLVM compiler to extract tuneable code regions and then apply
suggested configuration (by Autotuner) to find out the optimal configurations.
---
 clang/include/clang/Basic/CMakeLists.txt      |  15 ++
 .../clang/Basic/DiagnosticDriverKinds.td      |   9 ++
 .../clang/Basic/DiagnosticFrontendKinds.td    |   8 +
 clang/include/clang/Driver/CMakeLists.txt     |   6 +-
 clang/include/clang/Driver/Driver.h           |  36 +++++
 clang/include/clang/Driver/Options.td         |  13 ++
 clang/lib/CodeGen/BackendUtil.cpp             |  58 +++++++
 clang/lib/Driver/Driver.cpp                   |  82 ++++++++++
 clang/lib/Driver/ToolChains/Clang.cpp         |  21 +++
 clang/lib/Driver/ToolChains/CommonArgs.cpp    | 113 ++++++++++++++
 clang/lib/Driver/ToolChains/CommonArgs.h      |   8 +
 clang/lib/Driver/ToolChains/Gnu.cpp           |  34 ++++
 .../ExecuteCompilerInvocation.cpp             |  27 ++++
 .../autotune_datadir/baseline-config.yaml     |   9 ++
 .../autotune_datadir/random-config.yaml       |   9 ++
 .../BaselineConfig/apply-baseline-config.c    |  32 ++++
 .../test/Autotuning/Driver/Inputs/config.yaml |   3 +
 .../Autotuning/Driver/Inputs/template.yaml    |   9 ++
 .../Driver/autotune-generate-pipeline.c       | 146 ++++++++++++++++++
 .../Driver/autotune-pipeline-thin-lto.c       |  42 +++++
 .../Autotuning/Driver/autotune-pipeline.c     | 131 ++++++++++++++++
 .../test/Autotuning/GenerateOpp/generate.cpp  |  25 +++
 .../Inputs/template.yaml                      |   9 ++
 .../IncrementalCompilation/Inputs/test1.c     |   3 +
 .../IncrementalCompilation/Inputs/test2.c     |  17 ++
 .../IncrementalCompilation/Inputs/test3.c     |   6 +
 .../inc-compile-generate-input.cpp            |  44 ++++++
 .../Inputs/datadir/corse_grain_config.yaml    |   1 +
 .../LTO/Inputs/datadir/fine_grain_a.out.yaml  |   4 +
 .../LTO/Inputs/datadir/fine_grain_output.yaml |   1 +
 .../LTO/apply_config_coarse_grain.cpp         |  41 +++++
 .../LTO/apply_config_fine_grain.cpp           |  58 +++++++
 .../Autotuning/LTO/generate_opportunity.cpp   |  56 +++++++
 .../PhaseOrdering/Inputs/template.yaml        |   8 +
 .../Autotuning/PhaseOrdering/pass-order.cpp   |  48 ++++++
 42 files changed, 1170 insertions(+), 1 deletion(-)

diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt
index f010e04f62cd..e449d2790597 100644
--- a/clang/include/clang/Basic/CMakeLists.txt
+++ b/clang/include/clang/Basic/CMakeLists.txt
@@ -1,6 +1,12 @@
+set(CLANG_BASIC_OPTIONS)
+if(LLVM_ENABLE_AUTOTUNER)
+  list(APPEND CLANG_BASIC_OPTIONS "-DENABLE_AUTOTUNER")
+endif()
+
 macro(clang_diag_gen component)
   clang_tablegen(Diagnostic${component}Kinds.inc
     -gen-clang-diags-defs -clang-component=${component}
+    ${CLANG_BASIC_OPTIONS}
     SOURCE Diagnostic.td
     TARGET ClangDiagnostic${component})
 endmacro(clang_diag_gen)
@@ -18,20 +24,24 @@ clang_diag_gen(Refactoring)
 clang_diag_gen(Sema)
 clang_diag_gen(Serialization)
 clang_tablegen(DiagnosticGroups.inc -gen-clang-diag-groups
+  ${CLANG_BASIC_OPTIONS}
   SOURCE Diagnostic.td
   TARGET ClangDiagnosticGroups)
 
 clang_tablegen(DiagnosticIndexName.inc -gen-clang-diags-index-name
+  ${CLANG_BASIC_OPTIONS}
   SOURCE Diagnostic.td
   TARGET ClangDiagnosticIndexName)
 
 clang_tablegen(AttrList.inc -gen-clang-attr-list
   -I ${CMAKE_CURRENT_SOURCE_DIR}/../../
+  ${CLANG_BASIC_OPTIONS}
   SOURCE Attr.td
   TARGET ClangAttrList)
 
 clang_tablegen(AttrSubMatchRulesList.inc -gen-clang-attr-subject-match-rule-list
   -I ${CMAKE_CURRENT_SOURCE_DIR}/../../
+  ${CLANG_BASIC_OPTIONS}
   SOURCE Attr.td
   TARGET ClangAttrSubjectMatchRuleList)
 
@@ -43,6 +53,7 @@ clang_tablegen(AttrTokenKinds.inc -gen-clang-attr-token-kinds
 
 clang_tablegen(AttrHasAttributeImpl.inc -gen-clang-attr-has-attribute-impl
   -I ${CMAKE_CURRENT_SOURCE_DIR}/../../
+  ${CLANG_BASIC_OPTIONS}
   SOURCE Attr.td
   TARGET ClangAttrHasAttributeImpl
   )
@@ -67,15 +78,19 @@ clang_tablegen(arm_mve_builtin_aliases.inc -gen-arm-mve-builtin-aliases
   SOURCE arm_mve.td
   TARGET ClangARMMveBuiltinAliases)
 clang_tablegen(arm_sve_builtins.inc -gen-arm-sve-builtins
+  ${CLANG_BASIC_OPTIONS}
   SOURCE arm_sve.td
   TARGET ClangARMSveBuiltins)
 clang_tablegen(arm_sve_builtin_cg.inc -gen-arm-sve-builtin-codegen
+  ${CLANG_BASIC_OPTIONS}
   SOURCE arm_sve.td
   TARGET ClangARMSveBuiltinCG)
 clang_tablegen(arm_sve_typeflags.inc -gen-arm-sve-typeflags
+  ${CLANG_BASIC_OPTIONS}
   SOURCE arm_sve.td
   TARGET ClangARMSveTypeFlags)
 clang_tablegen(arm_sve_sema_rangechecks.inc -gen-arm-sve-sema-rangechecks
+  ${CLANG_BASIC_OPTIONS}
   SOURCE arm_sve.td
   TARGET ClangARMSveSemaRangeChecks)
 clang_tablegen(arm_sme_builtins.inc -gen-arm-sme-builtins
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 37586242953f..6b68bc458b93 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -248,6 +248,15 @@ def err_drv_cannot_read_config_file : Error<
   "cannot read configuration file '%0': %1">;
 def err_drv_arg_requires_bitcode_input: Error<
   "option '%0' requires input to be LLVM bitcode">;
+#ifdef ENABLE_AUTOTUNER
+def err_drv_autotune_generic : Error<"%0">;
+def err_drv_autotune_disabled_O0 : Error<
+  "-fautotune/-fautotune-generate should not be enabled at -O0">;
+def err_drv_autotune_incorrect_env : Error<
+  "incorrect argument '%0' in environment variable used">;
+def err_drv_autotune_no_filter_types : Error<
+  "no types added for filtering with %0">;
+#endif
 
 def err_target_unsupported_arch
   : Error<"the target architecture '%0' is not supported by the target '%1'">;
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 9ed9a88fa3d6..11022962ae9e 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -346,4 +346,12 @@ def warn_profile_data_misexpect : Warning<
 def err_extract_api_ignores_file_not_found :
   Error<"file '%0' specified by '--extract-api-ignores=' not found">, DefaultFatal;
 
+#ifdef ENABLE_AUTOTUNER
+let CategoryName = "AutoTuning Issues" in {
+def err_auto_tuning_error_reading : Error<"'%0'">;
+def err_auto_tuning_error_dumping : Error<"'%0'">;
+def err_unable_to_create_pass : Error<
+    "cannot create pass '%0' from AutoTuning input file">;
+} // end of autoTuning issue category
+#endif
 }
diff --git a/clang/include/clang/Driver/CMakeLists.txt b/clang/include/clang/Driver/CMakeLists.txt
index 8c0af1528a96..56fff6a2504e 100644
--- a/clang/include/clang/Driver/CMakeLists.txt
+++ b/clang/include/clang/Driver/CMakeLists.txt
@@ -8,7 +8,11 @@ endif()
 if (LLVM_ENABLE_CLASSIC_FLANG)
    list(APPEND CLANG_DRIVER_OPTIONS -DENABLE_CLASSIC_FLANG )
 endif()
- 
+
+if (LLVM_ENABLE_AUTOTUNER)
+   list(APPEND CLANG_DRIVER_OPTIONS "-DENABLE_AUTOTUNER" )
+endif()
+
 tablegen(LLVM Options.inc ${CLANG_DRIVER_OPTIONS} -gen-opt-parser-defs  )
 
 add_public_tablegen_target(ClangDriverOptions)
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index e3e98bad9912..dcecb473b516 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -72,6 +72,14 @@ enum ModuleHeaderMode {
   HeaderMode_System
 };
 
+#if defined(ENABLE_AUTOTUNER)
+enum AutoTuneKind {
+  AutoTuneNone,
+  AutoTuneGenerate,
+  AutoTuneNext,
+};
+#endif
+
 /// Driver - Encapsulate logic for constructing compilation processes
 /// from a set of gcc-driver-like command line arguments.
 class Driver {
@@ -119,6 +127,11 @@ class Driver {
   /// LTO mode selected via -f(no-offload-)?lto(=.*)? options.
   LTOKind OffloadLTOMode;
 
+#if defined(ENABLE_AUTOTUNER)
+  /// AutoTune mode selected via -fautotune or -fautotune-generate option
+  AutoTuneKind AutoTuneMode;
+#endif
+
 public:
   enum OpenMPRuntimeKind {
     /// An unknown OpenMP runtime. We can't generate effective OpenMP code
@@ -191,6 +204,21 @@ public:
   /// Information about the host which can be overridden by the user.
   std::string HostBits, HostMachine, HostSystem, HostRelease;
 
+#if defined(ENABLE_AUTOTUNER)
+  /// The path to the llvm-autotune data directory.
+  std::string AutoTuneDirDataPath;
+  /// Path for project base directory. Base directory is removed from absolute
+  /// path and relative path is used as (coarse-grain) code region name. This
+  /// allow to port a config file from one machine/location to another.
+  std::string AutoTuneProjectDir;
+
+  /// Whether to prepare the compiler to produce additional metadata
+  /// that will be consumed by Autotuner's ML model
+  bool IsMLTuningEnabled;
+
+  std::string AutoTuneOptions;
+#endif
+
   /// The file to log CC_PRINT_PROC_STAT_FILE output to, if enabled.
   std::string CCPrintStatReportFilename;
 
@@ -705,6 +733,14 @@ public:
     return IsOffload ? OffloadLTOMode : LTOMode;
   }
 
+#if defined(ENABLE_AUTOTUNER)
+  /// Returns true if we are performing any kind of AutoTune.
+  bool isUsingAutoTune() const { return AutoTuneMode != AutoTuneNone; }
+
+  /// Get the specific kind of AutoTune being performed.
+  AutoTuneKind getAutoTuneMode() const { return AutoTuneMode; }
+#endif
+
 private:
 
   /// Tries to load options from configuration files.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c5cc66c58f25..71d6ed66ab96 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1795,6 +1795,19 @@ def fmemory_profile_use_EQ : Joined<["-"], "fmemory-profile-use=">,
     Group<f_Group>, Flags<[CC1Option, CoreOption]>, MetaVarName<"<pathname>">,
     HelpText<"Use memory profile for profile-guided memory optimization">,
     MarshallingInfoString<CodeGenOpts<"MemoryProfileUsePath">>;
+#ifdef ENABLE_AUTOTUNER
+// Auto-tuning flags.
+def fautotune : Flag<["-"], "fautotune">, Group<f_Group>,
+  HelpText<"Auto-tune with the compiler configuration under 'autotune_datadir' (overridden by AUTOTUNE_DATADIR env var)">;
+def fautotune_EQ : Joined<["-"], "fautotune=">, Group<f_Group>,
+  HelpText<"Auto-tune with the compiler configuration of the specified id under 'autotune_datadir' (overridden by AUTOTUNE_DATADIR env var)">;
+def fautotune_generate : Flag<["-"], "fautotune-generate">, Group<f_Group>,
+  HelpText<"Generate initial compiler configuration for Function/Loop code regions under 'autotune_datadir' (overridden by AUTOTUNE_DATADIR env var)">;
+def fautotune_generate_EQ : CommaJoined<["-"], "fautotune-generate=">, Group<f_Group>,
+  HelpText<"Generate initial compiler configuration for the given comma-separated list of code regions under 'autotune_datadir' (overridden by AUTOTUNE_DATADIR env var)">, Values<"Other,Function,Loop,MachineBasicBlock">;
+def fautotune_rank : Flag<["-"], "fautotune-rank">, Group<f_Group>,
+  HelpText<"Generate files necessary for ML-guided ranking">;
+#endif
 
 #ifdef BUILD_FOR_OPENEULER
 def fgcc_compatible : Flag<["-"], "fgcc-compatible">, Group<f_Group>,
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index f962d60870d1..cef5e0d16ba7 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -88,6 +88,10 @@ using namespace llvm;
   llvm::PassPluginLibraryInfo get##Ext##PluginInfo();
 #include "llvm/Support/Extension.def"
 
+#if defined(ENABLE_AUTOTUNER)
+#include "llvm/Analysis/AutotuningDump.h"
+#endif
+
 namespace llvm {
 extern cl::opt<bool> DebugInfoCorrelate;
 
@@ -1021,6 +1025,27 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
           });
     }
 
+#if defined(ENABLE_AUTOTUNER)
+    bool Changed = false;
+    // If autotuning is enabled (for applying configuration), use AutoTuner
+    // generated pass ordering instead of passes in compilation pipeline. Passes
+    // before and after the compilation pipeline will be intact.
+    if (autotuning::Engine.isEnabled()) {
+      std::vector<std::string> PassesList;
+      Changed = autotuning::Engine.lookUpGlobalParams("OptPass", PassesList);
+      if (Changed && PassesList.size()) {
+        std::string PassPipeline = "";
+        for (auto PassName : PassesList)
+          PassPipeline.append(PassName + ",");
+        PassPipeline.pop_back();
+
+        if (auto Err = PB.parsePassPipeline(MPM, PassPipeline))
+          errs() << "AutoTuner: cannot add pass:" << toString(std::move(Err))
+                 << "\n";
+      }
+    }
+    if (!Changed) {
+#endif
     if (IsThinLTO || (IsLTO && CodeGenOpts.UnifiedLTO)) {
       MPM = PB.buildThinLTOPreLinkDefaultPipeline(Level);
     } else if (IsLTO) {
@@ -1028,6 +1053,9 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     } else {
       MPM = PB.buildPerModuleDefaultPipeline(Level);
     }
+#if defined(ENABLE_AUTOTUNER)
+    }
+#endif
   }
 
   // Add a verifier pass if requested. We don't have to do this if the action
@@ -1078,6 +1106,12 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     }
   }
 
+#if defined(ENABLE_AUTOTUNER)
+  // Please ensure this pass is added after all optimization passes.
+  if (autotuning::Engine.isEnabled())
+    MPM.addPass(RequireAnalysisPass<AutotuningDumpAnalysis, llvm::Module>());
+#endif
+
   // Now that we have all of the passes ready, run them.
   {
     PrettyStackTraceString CrashInfo("Optimizer");
@@ -1125,6 +1159,22 @@ void EmitAssemblyHelper::RunCodegenPipeline(
 void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
                                       std::unique_ptr<raw_pwrite_stream> OS) {
   TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
+
+#if defined(ENABLE_AUTOTUNER)
+  // AUTO-TUNING - auto-tuning initialization for this module.
+  // Initialize it before parsing command-line options because we want to
+  // overwrite the llvm options using the config file.
+  if (Error E = autotuning::Engine.init(TheModule->getModuleIdentifier())) {
+    Diags.Report(diag::err_auto_tuning_error_reading) << toString(std::move(E));
+    return;
+  }
+  if (autotuning::Engine.isEnabled() && autotuning::Engine.isParseInput() &&
+      (autotuning::Engine.LLVMParams.size() ||
+       autotuning::Engine.ProgramParams.size()))
+    llvm::cl::ParseAutoTunerOptions(autotuning::Engine.LLVMParams,
+                                    autotuning::Engine.ProgramParams);
+#endif
+
   setCommandLineOpts(CodeGenOpts);
 
   bool RequiresCodeGen = actionRequiresCodeGen(Action);
@@ -1142,6 +1192,14 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
   RunOptimizationPipeline(Action, OS, ThinLinkOS);
   RunCodegenPipeline(Action, OS, DwoOS);
 
+#if defined(ENABLE_AUTOTUNER)
+  // AUTO-TUNING - auto-tuning finalization for this module
+  if (Error E = autotuning::Engine.finalize()) {
+    Diags.Report(diag::err_auto_tuning_error_dumping) << toString(std::move(E));
+    return;
+  }
+#endif
+
   if (ThinLinkOS)
     ThinLinkOS->keep();
   if (DwoOS)
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 819d7703b2e7..bd9db7714f95 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -104,6 +104,14 @@
 #if LLVM_ON_UNIX
 #include <unistd.h> // getpid
 #endif
+#if defined(ENABLE_AUTOTUNER)
+// Constant definition for environment variable to enable AutoTuner and set
+// the mode to generate opportunities or apply configurations.
+const std::string AutoTuneModeStr = "AUTOTUNE_MODE";
+// Constant definition for environment variable to specify the project base
+// directory.
+const std::string AutoTunePrjDirStr = "AUTOTUNE_PROJECT_DIR";
+#endif
 
 using namespace clang::driver;
 using namespace clang;
@@ -200,6 +208,9 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
       SaveTemps(SaveTempsNone), BitcodeEmbed(EmbedNone),
       Offload(OffloadHostDevice), CXX20HeaderType(HeaderMode_None),
       ModulesModeCXX20(false), LTOMode(LTOK_None),
+#if defined(ENABLE_AUTOTUNER)
+      AutoTuneMode(AutoTuneNone),
+#endif
       ClangExecutable(ClangExecutable), SysRoot(DEFAULT_SYSROOT),
       DriverTitle(Title), CCCPrintBindings(false), CCPrintOptions(false),
       CCLogDiagnostics(false), CCGenDiagnostics(false),
@@ -1379,6 +1390,77 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
 
   setLTOMode(Args);
 
+#if defined(ENABLE_AUTOTUNER)
+  // Process -fautotune and -fautotune-generate flags.
+  bool IsAutoTuneGenerate = Args.hasArg(options::OPT_fautotune_generate,
+                                        options::OPT_fautotune_generate_EQ);
+  bool IsAutoTune =
+      Args.hasArg(options::OPT_fautotune, options::OPT_fautotune_EQ);
+  // Check if the environment variable AUTOTUNE_MODE is used instead of
+  // -fautotune-generate/-fautotune.
+  if (!IsAutoTuneGenerate && !IsAutoTune) {
+    if (std::optional<std::string> MaybeMode =
+            llvm::sys::Process::GetEnv(AutoTuneModeStr)) {
+      StringRef Mode = *MaybeMode;
+      StringRef OrgMode = *MaybeMode;
+      if (Mode.consume_front("-fautotune-generate")) {
+        if (Mode.empty() || Mode.startswith("="))
+          IsAutoTuneGenerate = true;
+        else
+          Diags.Report(diag::err_drv_autotune_incorrect_env) << OrgMode;
+      } else if (Mode.consume_front("-fautotune")) {
+        if (Mode.empty() || Mode.startswith("="))
+          IsAutoTune = true;
+        else
+          Diags.Report(diag::err_drv_autotune_incorrect_env) << OrgMode;
+      } else {
+        Diags.Report(diag::err_drv_autotune_incorrect_env) << OrgMode;
+      }
+
+      if (Mode.consume_front("=")) {
+        if (Mode.empty())
+          Diags.Report(diag::err_drv_autotune_no_filter_types)
+              << (IsAutoTuneGenerate ? "-fautotune-generate=" : "-fautotune=");
+
+        AutoTuneOptions = Mode.str();
+      }
+    }
+  }
+
+  IsMLTuningEnabled = Args.hasArg(options::OPT_fautotune_rank);
+
+  if (IsAutoTuneGenerate && IsAutoTune)
+    Diags.Report(diag::err_drv_argument_not_allowed_with)
+        << "-fautotune"
+        << "-fautotune-generate";
+
+  if (IsMLTuningEnabled && !(IsAutoTuneGenerate || IsAutoTune))
+    Diags.Report(diag::err_drv_argument_only_allowed_with)
+        << "-fautotune-rank"
+        << "-fautotune or -fautotune-generate";
+
+  if (IsAutoTuneGenerate || IsAutoTune) {
+    // Check if the environment variable AUTOTUNE_DATADIR is set.
+    if (std::optional<std::string> MaybePath =
+            llvm::sys::Process::GetEnv("AUTOTUNE_DATADIR"))
+      AutoTuneDirDataPath = *MaybePath;
+    else
+      AutoTuneDirDataPath = "autotune_datadir";
+
+    // Check if the environment variable AUTOTUNE_PROJECT_DIR is set.
+    if (std::optional<std::string> MaybeProjectDIR =
+            llvm::sys::Process::GetEnv(AutoTunePrjDirStr))
+      AutoTuneProjectDir = *MaybeProjectDIR;
+    else
+      AutoTuneProjectDir = "";
+
+    if (IsAutoTuneGenerate)
+      AutoTuneMode = AutoTuneGenerate;
+    if (IsAutoTune)
+      AutoTuneMode = AutoTuneNext;
+  }
+#endif
+
   // Process -fembed-bitcode= flags.
   if (Arg *A = Args.getLastArg(options::OPT_fembed_bitcode_EQ)) {
     StringRef Name = A->getValue();
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a4af991b5ff3..933661685117 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5990,6 +5990,27 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Triple.isNVPTX() && !Triple.isAMDGCN())
     addPGOAndCoverageFlags(TC, C, JA, Output, Args, SanitizeArgs, CmdArgs);
 
+#if defined(ENABLE_AUTOTUNER)
+  // Add Auto-tuning options.
+  if (C.getDriver().isUsingAutoTune()) {
+    Arg *A = Args.getLastArg(options::OPT_O_Group);
+    if (!A || A->getOption().matches(options::OPT_O0))
+      D.Diag(clang::diag::err_drv_autotune_disabled_O0);
+
+    // Enable debug info when Auto-tuning options are specified.
+    CmdArgs.push_back("-debug-info-kind=line-tables-only");
+    if (!D.AutoTuneProjectDir.empty()) {
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back(Args.MakeArgString(Twine("-autotuning-project-dir=") +
+                                           D.AutoTuneProjectDir));
+    }
+    if (D.getAutoTuneMode() == AutoTuneKind::AutoTuneGenerate)
+      AddAutoTuningOpportunities(Args, D, CmdArgs);
+    else if (D.getAutoTuneMode() == AutoTuneKind::AutoTuneNext)
+      AddAutoTuningInput(Args, D, CmdArgs);
+  }
+#endif
+
   Args.AddLastArg(CmdArgs, options::OPT_fclang_abi_compat_EQ);
 
   if (getLastProfileSampleUseArg(Args) &&
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 1ccc83a468ce..e01b21e102b1 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2429,6 +2429,119 @@ void tools::addMachineOutlinerArgs(const Driver &D,
   }
 }
 
+#if defined(ENABLE_AUTOTUNER)
+static bool isAcceptableThinLTOCodeRegion(StringRef CR) {
+  if ((CR.equals("CallSite") || CR.equals("Loop") || CR.equals("Function") ||
+       CR.equals("MachineBasicBlock")))
+    return false;
+  return true;
+}
+
+static bool processOpportunitiesOptions(StringRef CR, bool IsThinLTO,
+                                        std::string &CodeRegionsFilterStr) {
+  // Check if the argument has a valid value.
+  if (!(CR.equals("Other") || CR.equals("LLVMParam") || CR.equals("CallSite") ||
+        CR.equals("Function") || CR.equals("Loop") ||
+        CR.equals("MachineBasicBlock") || CR.equals("Switch") ||
+        CR.equals("ProgramParam")))
+    return false;
+
+  // Disable fine grain tuning for thin LTO during link time optimization.
+  if (IsThinLTO && !isAcceptableThinLTOCodeRegion(CR)) {
+    llvm::errs()
+        << "error: fine-grained autotuning not supported in ThinLTO mode\n";
+    return false;
+  }
+
+  if (!CodeRegionsFilterStr.empty())
+    CodeRegionsFilterStr += ',';
+  CodeRegionsFilterStr += CR;
+  return true;
+}
+
+// Add AutoTuner options for generating tuning opporutnities.
+// IsThinLTO will only be true during link time optimization for -flto=thin.
+void tools::AddAutoTuningOpportunities(const ArgList &Args, const Driver &D,
+                                       ArgStringList &CmdArgs, bool IsThinLTO) {
+  // Dump CodeRegions into opportunity files.
+  CmdArgs.push_back("-mllvm");
+  SmallString<128> OppPath = StringRef(D.AutoTuneDirDataPath);
+  llvm::sys::path::append(OppPath, "opp");
+  StringRef RawTypeFilterStr = D.AutoTuneOptions;
+  CmdArgs.push_back(Args.MakeArgString(Twine("-auto-tuning-opp=") + OppPath));
+  if (D.IsMLTuningEnabled) {
+    // Baseline config is -1
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back(Args.MakeArgString(Twine("-auto-tuning-config-id=-1")));
+  }
+  // Filter CodeRegions by type.
+  std::string CodeRegionsFilterStr;
+  if (Arg *A = Args.getLastArg(options::OPT_fautotune_generate_EQ)) {
+    for (StringRef CR : A->getValues()) {
+      if (!processOpportunitiesOptions(CR, IsThinLTO, CodeRegionsFilterStr))
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << A->getOption().getName() << CR;
+    }
+  } else if (!RawTypeFilterStr.empty()) {
+    SmallVector<StringRef, 8> TypeFilters;
+    RawTypeFilterStr.split(TypeFilters, ',');
+    for (StringRef CR : TypeFilters) {
+      if (!processOpportunitiesOptions(CR, IsThinLTO, CodeRegionsFilterStr))
+        D.Diag(diag::err_drv_unsupported_option_argument)
+            << "fautotune-generate" << CR;
+    }
+  } else {
+    if (IsThinLTO)
+      D.Diag(diag::err_drv_autotune_generic)
+          << "AutoTuner: no valid code region type specified for ThinLTO mode";
+    // Otherwise by default, dump CodeRegions of Function and Loop type.
+    CodeRegionsFilterStr = "CallSite,Function,Loop";
+  }
+  CmdArgs.push_back("-mllvm");
+  CmdArgs.push_back(
+      Args.MakeArgString("-auto-tuning-type-filter=" + CodeRegionsFilterStr));
+}
+
+static bool processInputOptions(StringRef Options, SmallString<128> &Path,
+                                const ArgList &Args, const Driver &D,
+                                llvm::opt::ArgStringList &CmdArgs) {
+  unsigned Value = 0;
+  // Check if the argument is an integer type.
+  if (Options.getAsInteger(10, Value))
+    return false;
+  llvm::sys::path::append(Path, "config-" + Twine(Value) + ".yaml");
+  if (D.IsMLTuningEnabled) {
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back(
+        Args.MakeArgString(Twine("-auto-tuning-config-id=" + Twine(Value))));
+  }
+  return true;
+}
+
+void tools::AddAutoTuningInput(const ArgList &Args, const Driver &D,
+                               llvm::opt::ArgStringList &CmdArgs) {
+  SmallString<128> InputPath = StringRef(D.AutoTuneDirDataPath);
+  StringRef RawOptionsStr = D.AutoTuneOptions;
+
+  if (Arg *A = Args.getLastArg(options::OPT_fautotune_EQ)) {
+    if (!processInputOptions(StringRef(A->getValue()), InputPath, Args, D,
+                             CmdArgs))
+      D.Diag(diag::err_drv_invalid_int_value)
+          << A->getAsString(Args) << A->getValue();
+  } else if (!RawOptionsStr.empty()) {
+    if (!processInputOptions(RawOptionsStr, InputPath, Args, D, CmdArgs))
+      D.Diag(diag::err_drv_invalid_int_value)
+          << "-fautotune=" + RawOptionsStr.str() << RawOptionsStr;
+  } else {
+    llvm::sys::path::append(InputPath, "config.yaml");
+  }
+  CmdArgs.push_back("-mllvm");
+  CmdArgs.push_back(
+      Args.MakeArgString(Twine("-auto-tuning-input=") + InputPath));
+  setenv("AUTOTUNE_INPUT", Args.MakeArgString(InputPath), 1);
+}
+#endif
+
 void tools::addOpenMPDeviceRTL(const Driver &D,
                                const llvm::opt::ArgList &DriverArgs,
                                llvm::opt::ArgStringList &CC1Args,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index bd5cb1bb866e..36103655c522 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -240,6 +240,14 @@ void addMachineOutlinerArgs(const Driver &D, const llvm::opt::ArgList &Args,
                             const llvm::Triple &Triple, bool IsLTO,
                             const StringRef PluginOptPrefix = "");
 
+#if defined(ENABLE_AUTOTUNER)
+void AddAutoTuningOpportunities(const llvm::opt::ArgList &Args, const Driver &D,
+                                llvm::opt::ArgStringList &CmdArgs,
+                                bool isThinLTO = false);
+void AddAutoTuningInput(const llvm::opt::ArgList &Args, const Driver &D,
+                        llvm::opt::ArgStringList &CmdArgs);
+#endif
+
 void addOpenMPDeviceRTL(const Driver &D, const llvm::opt::ArgList &DriverArgs,
                         llvm::opt::ArgStringList &CC1Args,
                         StringRef BitcodeSuffix, const llvm::Triple &Triple);
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 42050dced99c..91a9eda9d78c 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -682,6 +682,40 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_T);
 
+#if defined(ENABLE_AUTOTUNER)
+  // AutoTuner related features will only be enabled for LTO build during
+  // linking phase. Otherwise, non LTO build will require lld linker
+  // unnecessarily (other linkers do not support AutoTuner).
+  if (D.isUsingAutoTune() && D.isUsingLTO()) {
+    bool LinkerIsLLD = false;
+    (void) ToolChain.GetLinkerPath(&LinkerIsLLD);
+    // AutoTuner support is only available for LLD Linker.
+    if (!LinkerIsLLD)
+      D.Diag(clang::diag::err_drv_lto_without_lld);
+
+    bool IsThinLTO = D.getLTOMode() == LTOK_Thin;
+    if (!D.AutoTuneProjectDir.empty()) {
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back(Args.MakeArgString(Twine("-autotuning-project-dir=") +
+                                           D.AutoTuneProjectDir));
+    }
+    // Enable tuning of callsites cause all of the callsites will have local
+    // linkage during LTO and they are not tuned by default.
+    CmdArgs.push_back(Args.MakeArgString("-mllvm"));
+    CmdArgs.push_back(
+        Args.MakeArgString("-auto-tuning-enable-local-callsite-tuning=true"));
+    if (D.getAutoTuneMode() == AutoTuneKind::AutoTuneGenerate) {
+      AddAutoTuningOpportunities(Args, D, CmdArgs, IsThinLTO);
+    } else if (D.getAutoTuneMode() == AutoTuneKind::AutoTuneNext) {
+      AddAutoTuningInput(Args, D, CmdArgs);
+      if (IsThinLTO) {
+        CmdArgs.push_back("-mllvm");
+        CmdArgs.push_back("-autotuning-thin-lto=true");
+      }
+    }
+  }
+#endif
+
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 310f67774a66..92beeef9bd5e 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -222,6 +222,33 @@ bool ExecuteCompilerInvocation(CompilerInstance *Clang) {
   // This should happen AFTER plugins have been loaded!
   if (!Clang->getFrontendOpts().LLVMArgs.empty()) {
     unsigned NumArgs = Clang->getFrontendOpts().LLVMArgs.size();
+#if defined(ENABLE_AUTOTUNER)
+    // Both incremental compilation (for AutoTuner) and 'opt-bisect-limit'
+    // changes the behavior of compilation pipeline. If incremental compilation
+    // is used along with 'opt-bisect-limit' then 'opt-bisect-limit' is
+    // preferred and incremental compilation is disabled.
+    unsigned BisectLimitFound = 0;
+    unsigned CompileModeFound = 0;
+    for (unsigned Idx = 0; Idx != NumArgs; ++Idx) {
+      if (Clang->getFrontendOpts().LLVMArgs[Idx].find("-opt-bisect-limit=") !=
+          std::string::npos)
+        BisectLimitFound = Idx;
+      if (Clang->getFrontendOpts().LLVMArgs[Idx].find(
+              "-auto-tuning-compile-mode=") != std::string::npos)
+        CompileModeFound = Idx;
+      if (BisectLimitFound && CompileModeFound)
+        break;
+    }
+    if (BisectLimitFound && CompileModeFound &&
+        Clang->getFrontendOpts().LLVMArgs[CompileModeFound].compare(
+            "-auto-tuning-compile-mode=Inactive") != 0) {
+      Clang->getFrontendOpts().LLVMArgs[CompileModeFound] =
+          "-auto-tuning-compile-mode=Inactive";
+      llvm::errs() << "AutoTunerCompile: Incremental compilation cannot work "
+                      "with '-opt-bisect-limit' flag.\n"
+                      "Disabling incremental compilation.\n";
+    }
+#endif
     auto Args = std::make_unique<const char*[]>(NumArgs + 2);
     Args[0] = "clang (LLVM option parsing)";
     for (unsigned i = 0; i != NumArgs; ++i)
-- 
2.33.0