Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 1 | //===- GlobalMerge.cpp - Internal globals merging -------------------------===// |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 9 | // |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 10 | // This pass merges globals with internal linkage into one. This way all the |
| 11 | // globals which were merged into a biggest one can be addressed using offsets |
| 12 | // from the same base pointer (no need for separate base pointer for each of the |
| 13 | // global). Such a transformation can significantly reduce the register pressure |
| 14 | // when many globals are involved. |
| 15 | // |
Nadav Rotem | a94d6e8 | 2012-07-24 10:51:42 +0000 | [diff] [blame] | 16 | // For example, consider the code which touches several global variables at |
Eric Christopher | a99c3e9 | 2010-09-28 04:18:29 +0000 | [diff] [blame] | 17 | // once: |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 18 | // |
| 19 | // static int foo[N], bar[N], baz[N]; |
| 20 | // |
| 21 | // for (i = 0; i < N; ++i) { |
| 22 | // foo[i] = bar[i] * baz[i]; |
| 23 | // } |
| 24 | // |
| 25 | // On ARM the addresses of 3 arrays should be kept in the registers, thus |
| 26 | // this code has quite large register pressure (loop body): |
| 27 | // |
| 28 | // ldr r1, [r5], #4 |
| 29 | // ldr r2, [r6], #4 |
| 30 | // mul r1, r2, r1 |
| 31 | // str r1, [r0], #4 |
| 32 | // |
| 33 | // Pass converts the code to something like: |
| 34 | // |
| 35 | // static struct { |
| 36 | // int foo[N]; |
| 37 | // int bar[N]; |
| 38 | // int baz[N]; |
| 39 | // } merged; |
| 40 | // |
| 41 | // for (i = 0; i < N; ++i) { |
| 42 | // merged.foo[i] = merged.bar[i] * merged.baz[i]; |
| 43 | // } |
| 44 | // |
| 45 | // and in ARM code this becomes: |
| 46 | // |
| 47 | // ldr r0, [r5, #40] |
| 48 | // ldr r1, [r5, #80] |
| 49 | // mul r0, r1, r0 |
| 50 | // str r0, [r5], #4 |
| 51 | // |
| 52 | // note that we saved 2 registers here almostly "for free". |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 53 | // |
| 54 | // However, merging globals can have tradeoffs: |
| 55 | // - it confuses debuggers, tools, and users |
| 56 | // - it makes linker optimizations less useful (order files, LOHs, ...) |
| 57 | // - it forces usage of indexed addressing (which isn't necessarily "free") |
| 58 | // - it can increase register pressure when the uses are disparate enough. |
Fangrui Song | af7b183 | 2018-07-30 19:41:25 +0000 | [diff] [blame] | 59 | // |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 60 | // We use heuristics to discover the best global grouping we can (cf cl::opts). |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 61 | // |
Eric Christopher | a99c3e9 | 2010-09-28 04:18:29 +0000 | [diff] [blame] | 62 | // ===---------------------------------------------------------------------===// |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 63 | |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 64 | #include "llvm/ADT/BitVector.h" |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 65 | #include "llvm/ADT/DenseMap.h" |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 66 | #include "llvm/ADT/SmallPtrSet.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 67 | #include "llvm/ADT/SmallVector.h" |
Chandler Carruth | d04a8d4 | 2012-12-03 16:50:05 +0000 | [diff] [blame] | 68 | #include "llvm/ADT/Statistic.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 69 | #include "llvm/ADT/StringRef.h" |
| 70 | #include "llvm/ADT/Triple.h" |
| 71 | #include "llvm/ADT/Twine.h" |
Chandler Carruth | 1b27914 | 2015-01-14 11:23:27 +0000 | [diff] [blame] | 72 | #include "llvm/CodeGen/Passes.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 73 | #include "llvm/IR/BasicBlock.h" |
Chandler Carruth | 0b8c9a8 | 2013-01-02 11:36:10 +0000 | [diff] [blame] | 74 | #include "llvm/IR/Constants.h" |
| 75 | #include "llvm/IR/DataLayout.h" |
| 76 | #include "llvm/IR/DerivedTypes.h" |
| 77 | #include "llvm/IR/Function.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 78 | #include "llvm/IR/GlobalAlias.h" |
| 79 | #include "llvm/IR/GlobalValue.h" |
Chandler Carruth | 0b8c9a8 | 2013-01-02 11:36:10 +0000 | [diff] [blame] | 80 | #include "llvm/IR/GlobalVariable.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 81 | #include "llvm/IR/Instruction.h" |
Chandler Carruth | 0b8c9a8 | 2013-01-02 11:36:10 +0000 | [diff] [blame] | 82 | #include "llvm/IR/Module.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 83 | #include "llvm/IR/Type.h" |
| 84 | #include "llvm/IR/Use.h" |
| 85 | #include "llvm/IR/User.h" |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 86 | #include "llvm/Pass.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 87 | #include "llvm/Support/Casting.h" |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 88 | #include "llvm/Support/CommandLine.h" |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 89 | #include "llvm/Support/Debug.h" |
| 90 | #include "llvm/Support/raw_ostream.h" |
David Blaikie | fe42bd5 | 2018-03-23 23:58:19 +0000 | [diff] [blame] | 91 | #include "llvm/Target/TargetLoweringObjectFile.h" |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 92 | #include "llvm/Target/TargetMachine.h" |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 93 | #include <algorithm> |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 94 | #include <cassert> |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 95 | #include <cstddef> |
David Blaikie | e3a9b4c | 2017-11-17 01:07:10 +0000 | [diff] [blame] | 96 | #include <cstdint> |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 97 | #include <string> |
| 98 | #include <vector> |
| 99 | |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 100 | using namespace llvm; |
| 101 | |
Chandler Carruth | 7962dbd | 2014-04-22 02:55:47 +0000 | [diff] [blame] | 102 | #define DEBUG_TYPE "global-merge" |
| 103 | |
Ahmed Bougacha | d206933 | 2015-04-11 00:06:36 +0000 | [diff] [blame] | 104 | // FIXME: This is only useful as a last-resort way to disable the pass. |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 105 | static cl::opt<bool> |
Jiangning Liu | 4c3b0b7 | 2014-06-11 06:35:26 +0000 | [diff] [blame] | 106 | EnableGlobalMerge("enable-global-merge", cl::Hidden, |
Ahmed Bougacha | d206933 | 2015-04-11 00:06:36 +0000 | [diff] [blame] | 107 | cl::desc("Enable the global merge pass"), |
Tim Northover | 4bcb985 | 2014-02-18 11:17:29 +0000 | [diff] [blame] | 108 | cl::init(true)); |
| 109 | |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 110 | static cl::opt<unsigned> |
| 111 | GlobalMergeMaxOffset("global-merge-max-offset", cl::Hidden, |
| 112 | cl::desc("Set maximum offset for global merge pass"), |
| 113 | cl::init(0)); |
| 114 | |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 115 | static cl::opt<bool> GlobalMergeGroupByUse( |
| 116 | "global-merge-group-by-use", cl::Hidden, |
| 117 | cl::desc("Improve global merge pass to look at uses"), cl::init(true)); |
| 118 | |
| 119 | static cl::opt<bool> GlobalMergeIgnoreSingleUse( |
| 120 | "global-merge-ignore-single-use", cl::Hidden, |
| 121 | cl::desc("Improve global merge pass to ignore globals only used alone"), |
| 122 | cl::init(true)); |
| 123 | |
Tim Northover | 4bcb985 | 2014-02-18 11:17:29 +0000 | [diff] [blame] | 124 | static cl::opt<bool> |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 125 | EnableGlobalMergeOnConst("global-merge-on-const", cl::Hidden, |
Jakub Staszak | dca13e0 | 2013-07-22 21:11:30 +0000 | [diff] [blame] | 126 | cl::desc("Enable global merge pass on constants"), |
| 127 | cl::init(false)); |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 128 | |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 129 | // FIXME: this could be a transitional option, and we probably need to remove |
| 130 | // it if only we are sure this optimization could always benefit all targets. |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 131 | static cl::opt<cl::boolOrDefault> |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 132 | EnableGlobalMergeOnExternal("global-merge-on-external", cl::Hidden, |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 133 | cl::desc("Enable global merge pass on external linkage")); |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 134 | |
Eric Christopher | 308458a | 2015-02-23 19:28:45 +0000 | [diff] [blame] | 135 | STATISTIC(NumMerged, "Number of globals merged"); |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 136 | |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 137 | namespace { |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 138 | |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 139 | class GlobalMerge : public FunctionPass { |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 140 | const TargetMachine *TM = nullptr; |
| 141 | |
Eric Christopher | 308458a | 2015-02-23 19:28:45 +0000 | [diff] [blame] | 142 | // FIXME: Infer the maximum possible offset depending on the actual users |
| 143 | // (these max offsets are different for the users inside Thumb or ARM |
| 144 | // functions), see the code that passes in the offset in the ARM backend |
| 145 | // for more information. |
| 146 | unsigned MaxOffset; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 147 | |
Ahmed Bougacha | 0d9335e | 2015-06-04 20:39:23 +0000 | [diff] [blame] | 148 | /// Whether we should try to optimize for size only. |
| 149 | /// Currently, this applies a dead simple heuristic: only consider globals |
| 150 | /// used in minsize functions for merging. |
| 151 | /// FIXME: This could learn about optsize, and be used in the cost model. |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 152 | bool OnlyOptimizeForSize = false; |
Ahmed Bougacha | 0d9335e | 2015-06-04 20:39:23 +0000 | [diff] [blame] | 153 | |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 154 | /// Whether we should merge global variables that have external linkage. |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 155 | bool MergeExternalGlobals = false; |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 156 | |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 157 | bool IsMachO; |
| 158 | |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 159 | bool doMerge(SmallVectorImpl<GlobalVariable*> &Globals, |
Silviu Baranga | e971659 | 2013-01-07 12:31:25 +0000 | [diff] [blame] | 160 | Module &M, bool isConst, unsigned AddrSpace) const; |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 161 | |
Adrian Prantl | 26b584c | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 162 | /// Merge everything in \p Globals for which the corresponding bit |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 163 | /// in \p GlobalSet is set. |
David Blaikie | f47cf24 | 2015-08-21 22:19:06 +0000 | [diff] [blame] | 164 | bool doMerge(const SmallVectorImpl<GlobalVariable *> &Globals, |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 165 | const BitVector &GlobalSet, Module &M, bool isConst, |
| 166 | unsigned AddrSpace) const; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 167 | |
Adrian Prantl | 26b584c | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 168 | /// Check if the given variable has been identified as must keep |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 169 | /// \pre setMustKeepGlobalVariables must have been called on the Module that |
| 170 | /// contains GV |
| 171 | bool isMustKeepGlobalVariable(const GlobalVariable *GV) const { |
| 172 | return MustKeepGlobalVariables.count(GV); |
| 173 | } |
| 174 | |
| 175 | /// Collect every variables marked as "used" or used in a landing pad |
| 176 | /// instruction for this Module. |
| 177 | void setMustKeepGlobalVariables(Module &M); |
| 178 | |
| 179 | /// Collect every variables marked as "used" |
Eli Friedman | 261e479 | 2018-07-25 22:03:35 +0000 | [diff] [blame] | 180 | void collectUsedGlobalVariables(Module &M, StringRef Name); |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 181 | |
Quentin Colombet | 9deb917 | 2013-03-19 21:46:49 +0000 | [diff] [blame] | 182 | /// Keep track of the GlobalVariable that must not be merged away |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 183 | SmallPtrSet<const GlobalVariable *, 16> MustKeepGlobalVariables; |
| 184 | |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 185 | public: |
| 186 | static char ID; // Pass identification, replacement for typeid. |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 187 | |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 188 | explicit GlobalMerge() |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 189 | : FunctionPass(ID), MaxOffset(GlobalMergeMaxOffset) { |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 190 | initializeGlobalMergePass(*PassRegistry::getPassRegistry()); |
| 191 | } |
| 192 | |
| 193 | explicit GlobalMerge(const TargetMachine *TM, unsigned MaximalOffset, |
| 194 | bool OnlyOptimizeForSize, bool MergeExternalGlobals) |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 195 | : FunctionPass(ID), TM(TM), MaxOffset(MaximalOffset), |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 196 | OnlyOptimizeForSize(OnlyOptimizeForSize), |
| 197 | MergeExternalGlobals(MergeExternalGlobals) { |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 198 | initializeGlobalMergePass(*PassRegistry::getPassRegistry()); |
| 199 | } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 200 | |
Craig Topper | 7b62be2 | 2014-03-05 09:10:37 +0000 | [diff] [blame] | 201 | bool doInitialization(Module &M) override; |
| 202 | bool runOnFunction(Function &F) override; |
| 203 | bool doFinalization(Module &M) override; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 204 | |
Mehdi Amini | 67f335d | 2016-10-01 02:56:57 +0000 | [diff] [blame] | 205 | StringRef getPassName() const override { return "Merge internal globals"; } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 206 | |
Craig Topper | 7b62be2 | 2014-03-05 09:10:37 +0000 | [diff] [blame] | 207 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 208 | AU.setPreservesCFG(); |
| 209 | FunctionPass::getAnalysisUsage(AU); |
| 210 | } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 211 | }; |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 212 | |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 213 | } // end anonymous namespace |
| 214 | |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 215 | char GlobalMerge::ID = 0; |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 216 | |
Matthias Braun | 94c4904 | 2017-05-25 21:26:32 +0000 | [diff] [blame] | 217 | INITIALIZE_PASS(GlobalMerge, DEBUG_TYPE, "Merge global variables", false, false) |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 218 | |
| 219 | bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals, |
Silviu Baranga | e971659 | 2013-01-07 12:31:25 +0000 | [diff] [blame] | 220 | Module &M, bool isConst, unsigned AddrSpace) const { |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 221 | auto &DL = M.getDataLayout(); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 222 | // FIXME: Find better heuristics |
David Blaikie | f60104e | 2015-08-21 22:00:44 +0000 | [diff] [blame] | 223 | std::stable_sort(Globals.begin(), Globals.end(), |
| 224 | [&DL](const GlobalVariable *GV1, const GlobalVariable *GV2) { |
| 225 | return DL.getTypeAllocSize(GV1->getValueType()) < |
| 226 | DL.getTypeAllocSize(GV2->getValueType()); |
| 227 | }); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 228 | |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 229 | // If we want to just blindly group all globals together, do so. |
| 230 | if (!GlobalMergeGroupByUse) { |
| 231 | BitVector AllGlobals(Globals.size()); |
| 232 | AllGlobals.set(); |
| 233 | return doMerge(Globals, AllGlobals, M, isConst, AddrSpace); |
| 234 | } |
| 235 | |
| 236 | // If we want to be smarter, look at all uses of each global, to try to |
| 237 | // discover all sets of globals used together, and how many times each of |
Benjamin Kramer | d3c712e | 2015-08-08 18:27:36 +0000 | [diff] [blame] | 238 | // these sets occurred. |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 239 | // |
| 240 | // Keep this reasonably efficient, by having an append-only list of all sets |
| 241 | // discovered so far (UsedGlobalSet), and mapping each "together-ness" unit of |
| 242 | // code (currently, a Function) to the set of globals seen so far that are |
| 243 | // used together in that unit (GlobalUsesByFunction). |
| 244 | // |
Haicheng Wu | 7d80a62 | 2018-04-26 17:56:50 +0000 | [diff] [blame] | 245 | // When we look at the Nth global, we know that any new set is either: |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 246 | // - the singleton set {N}, containing this global only, or |
| 247 | // - the union of {N} and a previously-discovered set, containing some |
| 248 | // combination of the previous N-1 globals. |
| 249 | // Using that knowledge, when looking at the Nth global, we can keep: |
| 250 | // - a reference to the singleton set {N} (CurGVOnlySetIdx) |
| 251 | // - a list mapping each previous set to its union with {N} (EncounteredUGS), |
| 252 | // if it actually occurs. |
| 253 | |
| 254 | // We keep track of the sets of globals used together "close enough". |
| 255 | struct UsedGlobalSet { |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 256 | BitVector Globals; |
Eugene Zelenko | 3840975 | 2017-09-22 23:46:57 +0000 | [diff] [blame] | 257 | unsigned UsageCount = 1; |
| 258 | |
| 259 | UsedGlobalSet(size_t Size) : Globals(Size) {} |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 260 | }; |
| 261 | |
| 262 | // Each set is unique in UsedGlobalSets. |
| 263 | std::vector<UsedGlobalSet> UsedGlobalSets; |
| 264 | |
| 265 | // Avoid repeating the create-global-set pattern. |
| 266 | auto CreateGlobalSet = [&]() -> UsedGlobalSet & { |
| 267 | UsedGlobalSets.emplace_back(Globals.size()); |
| 268 | return UsedGlobalSets.back(); |
| 269 | }; |
| 270 | |
| 271 | // The first set is the empty set. |
| 272 | CreateGlobalSet().UsageCount = 0; |
| 273 | |
| 274 | // We define "close enough" to be "in the same function". |
| 275 | // FIXME: Grouping uses by function is way too aggressive, so we should have |
| 276 | // a better metric for distance between uses. |
| 277 | // The obvious alternative would be to group by BasicBlock, but that's in |
| 278 | // turn too conservative.. |
| 279 | // Anything in between wouldn't be trivial to compute, so just stick with |
| 280 | // per-function grouping. |
| 281 | |
| 282 | // The value type is an index into UsedGlobalSets. |
| 283 | // The default (0) conveniently points to the empty set. |
| 284 | DenseMap<Function *, size_t /*UsedGlobalSetIdx*/> GlobalUsesByFunction; |
| 285 | |
| 286 | // Now, look at each merge-eligible global in turn. |
| 287 | |
| 288 | // Keep track of the sets we already encountered to which we added the |
| 289 | // current global. |
| 290 | // Each element matches the same-index element in UsedGlobalSets. |
| 291 | // This lets us efficiently tell whether a set has already been expanded to |
| 292 | // include the current global. |
| 293 | std::vector<size_t> EncounteredUGS; |
| 294 | |
| 295 | for (size_t GI = 0, GE = Globals.size(); GI != GE; ++GI) { |
| 296 | GlobalVariable *GV = Globals[GI]; |
| 297 | |
| 298 | // Reset the encountered sets for this global... |
| 299 | std::fill(EncounteredUGS.begin(), EncounteredUGS.end(), 0); |
| 300 | // ...and grow it in case we created new sets for the previous global. |
| 301 | EncounteredUGS.resize(UsedGlobalSets.size()); |
| 302 | |
| 303 | // We might need to create a set that only consists of the current global. |
| 304 | // Keep track of its index into UsedGlobalSets. |
| 305 | size_t CurGVOnlySetIdx = 0; |
| 306 | |
| 307 | // For each global, look at all its Uses. |
| 308 | for (auto &U : GV->uses()) { |
| 309 | // This Use might be a ConstantExpr. We're interested in Instruction |
| 310 | // users, so look through ConstantExpr... |
| 311 | Use *UI, *UE; |
| 312 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U.getUser())) { |
Oliver Stannard | 7c40b1a | 2015-06-08 16:55:31 +0000 | [diff] [blame] | 313 | if (CE->use_empty()) |
| 314 | continue; |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 315 | UI = &*CE->use_begin(); |
| 316 | UE = nullptr; |
| 317 | } else if (isa<Instruction>(U.getUser())) { |
| 318 | UI = &U; |
| 319 | UE = UI->getNext(); |
| 320 | } else { |
| 321 | continue; |
| 322 | } |
| 323 | |
| 324 | // ...to iterate on all the instruction users of the global. |
| 325 | // Note that we iterate on Uses and not on Users to be able to getNext(). |
| 326 | for (; UI != UE; UI = UI->getNext()) { |
| 327 | Instruction *I = dyn_cast<Instruction>(UI->getUser()); |
| 328 | if (!I) |
| 329 | continue; |
| 330 | |
| 331 | Function *ParentFn = I->getParent()->getParent(); |
Ahmed Bougacha | 0d9335e | 2015-06-04 20:39:23 +0000 | [diff] [blame] | 332 | |
| 333 | // If we're only optimizing for size, ignore non-minsize functions. |
Sanjay Patel | 08c1ad0 | 2015-08-18 16:44:23 +0000 | [diff] [blame] | 334 | if (OnlyOptimizeForSize && !ParentFn->optForMinSize()) |
Ahmed Bougacha | 0d9335e | 2015-06-04 20:39:23 +0000 | [diff] [blame] | 335 | continue; |
| 336 | |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 337 | size_t UGSIdx = GlobalUsesByFunction[ParentFn]; |
| 338 | |
| 339 | // If this is the first global the basic block uses, map it to the set |
| 340 | // consisting of this global only. |
| 341 | if (!UGSIdx) { |
| 342 | // If that set doesn't exist yet, create it. |
| 343 | if (!CurGVOnlySetIdx) { |
| 344 | CurGVOnlySetIdx = UsedGlobalSets.size(); |
| 345 | CreateGlobalSet().Globals.set(GI); |
| 346 | } else { |
| 347 | ++UsedGlobalSets[CurGVOnlySetIdx].UsageCount; |
| 348 | } |
| 349 | |
| 350 | GlobalUsesByFunction[ParentFn] = CurGVOnlySetIdx; |
| 351 | continue; |
| 352 | } |
| 353 | |
| 354 | // If we already encountered this BB, just increment the counter. |
| 355 | if (UsedGlobalSets[UGSIdx].Globals.test(GI)) { |
| 356 | ++UsedGlobalSets[UGSIdx].UsageCount; |
| 357 | continue; |
| 358 | } |
| 359 | |
| 360 | // If not, the previous set wasn't actually used in this function. |
| 361 | --UsedGlobalSets[UGSIdx].UsageCount; |
| 362 | |
| 363 | // If we already expanded the previous set to include this global, just |
| 364 | // reuse that expanded set. |
| 365 | if (size_t ExpandedIdx = EncounteredUGS[UGSIdx]) { |
| 366 | ++UsedGlobalSets[ExpandedIdx].UsageCount; |
| 367 | GlobalUsesByFunction[ParentFn] = ExpandedIdx; |
| 368 | continue; |
| 369 | } |
| 370 | |
| 371 | // If not, create a new set consisting of the union of the previous set |
| 372 | // and this global. Mark it as encountered, so we can reuse it later. |
| 373 | GlobalUsesByFunction[ParentFn] = EncounteredUGS[UGSIdx] = |
| 374 | UsedGlobalSets.size(); |
| 375 | |
| 376 | UsedGlobalSet &NewUGS = CreateGlobalSet(); |
| 377 | NewUGS.Globals.set(GI); |
| 378 | NewUGS.Globals |= UsedGlobalSets[UGSIdx].Globals; |
| 379 | } |
| 380 | } |
| 381 | } |
| 382 | |
| 383 | // Now we found a bunch of sets of globals used together. We accumulated |
| 384 | // the number of times we encountered the sets (i.e., the number of blocks |
| 385 | // that use that exact set of globals). |
| 386 | // |
| 387 | // Multiply that by the size of the set to give us a crude profitability |
| 388 | // metric. |
Mandeep Singh Grang | d28baf6 | 2017-11-09 18:05:17 +0000 | [diff] [blame] | 389 | std::stable_sort(UsedGlobalSets.begin(), UsedGlobalSets.end(), |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 390 | [](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) { |
| 391 | return UGS1.Globals.count() * UGS1.UsageCount < |
| 392 | UGS2.Globals.count() * UGS2.UsageCount; |
| 393 | }); |
| 394 | |
| 395 | // We can choose to merge all globals together, but ignore globals never used |
| 396 | // with another global. This catches the obviously non-profitable cases of |
| 397 | // having a single global, but is aggressive enough for any other case. |
| 398 | if (GlobalMergeIgnoreSingleUse) { |
| 399 | BitVector AllGlobals(Globals.size()); |
| 400 | for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) { |
| 401 | const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1]; |
| 402 | if (UGS.UsageCount == 0) |
| 403 | continue; |
| 404 | if (UGS.Globals.count() > 1) |
| 405 | AllGlobals |= UGS.Globals; |
| 406 | } |
| 407 | return doMerge(Globals, AllGlobals, M, isConst, AddrSpace); |
| 408 | } |
| 409 | |
| 410 | // Starting from the sets with the best (=biggest) profitability, find a |
| 411 | // good combination. |
| 412 | // The ideal (and expensive) solution can only be found by trying all |
| 413 | // combinations, looking for the one with the best profitability. |
| 414 | // Don't be smart about it, and just pick the first compatible combination, |
| 415 | // starting with the sets with the best profitability. |
| 416 | BitVector PickedGlobals(Globals.size()); |
| 417 | bool Changed = false; |
| 418 | |
| 419 | for (size_t i = 0, e = UsedGlobalSets.size(); i != e; ++i) { |
| 420 | const UsedGlobalSet &UGS = UsedGlobalSets[e - i - 1]; |
| 421 | if (UGS.UsageCount == 0) |
| 422 | continue; |
| 423 | if (PickedGlobals.anyCommon(UGS.Globals)) |
| 424 | continue; |
| 425 | PickedGlobals |= UGS.Globals; |
| 426 | // If the set only contains one global, there's no point in merging. |
| 427 | // Ignore the global for inclusion in other sets though, so keep it in |
| 428 | // PickedGlobals. |
| 429 | if (UGS.Globals.count() < 2) |
| 430 | continue; |
| 431 | Changed |= doMerge(Globals, UGS.Globals, M, isConst, AddrSpace); |
| 432 | } |
| 433 | |
| 434 | return Changed; |
| 435 | } |
| 436 | |
David Blaikie | f47cf24 | 2015-08-21 22:19:06 +0000 | [diff] [blame] | 437 | bool GlobalMerge::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals, |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 438 | const BitVector &GlobalSet, Module &M, bool isConst, |
| 439 | unsigned AddrSpace) const { |
David Blaikie | f47cf24 | 2015-08-21 22:19:06 +0000 | [diff] [blame] | 440 | assert(Globals.size() > 1); |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 441 | |
Chris Lattner | db125cf | 2011-07-18 04:54:35 +0000 | [diff] [blame] | 442 | Type *Int32Ty = Type::getInt32Ty(M.getContext()); |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 443 | Type *Int8Ty = Type::getInt8Ty(M.getContext()); |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 444 | auto &DL = M.getDataLayout(); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 445 | |
Nicola Zaghen | 0818e78 | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 446 | LLVM_DEBUG(dbgs() << " Trying to merge set, starts with #" |
| 447 | << GlobalSet.find_first() << "\n"); |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 448 | |
Haicheng Wu | b508121 | 2018-05-19 18:00:02 +0000 | [diff] [blame] | 449 | bool Changed = false; |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 450 | ssize_t i = GlobalSet.find_first(); |
| 451 | while (i != -1) { |
| 452 | ssize_t j = 0; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 453 | uint64_t MergedSize = 0; |
Jay Foad | 5fdd6c8 | 2011-07-12 14:06:48 +0000 | [diff] [blame] | 454 | std::vector<Type*> Tys; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 455 | std::vector<Constant*> Inits; |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 456 | std::vector<unsigned> StructIdxs; |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 457 | |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 458 | bool HasExternal = false; |
Adrian Prantl | feb16c8 | 2016-11-11 22:09:25 +0000 | [diff] [blame] | 459 | StringRef FirstExternalName; |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 460 | unsigned MaxAlign = 1; |
| 461 | unsigned CurIdx = 0; |
Ahmed Bougacha | 35e8055 | 2015-04-18 01:21:58 +0000 | [diff] [blame] | 462 | for (j = i; j != -1; j = GlobalSet.find_next(j)) { |
David Blaikie | f60104e | 2015-08-21 22:00:44 +0000 | [diff] [blame] | 463 | Type *Ty = Globals[j]->getValueType(); |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 464 | |
Eli Friedman | 47dc019 | 2018-08-29 23:46:26 +0000 | [diff] [blame] | 465 | // Make sure we use the same alignment AsmPrinter would use. |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 466 | unsigned Align = DL.getPreferredAlignment(Globals[j]); |
| 467 | unsigned Padding = alignTo(MergedSize, Align) - MergedSize; |
| 468 | MergedSize += Padding; |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 469 | MergedSize += DL.getTypeAllocSize(Ty); |
Bob Wilson | 619a372 | 2010-11-17 21:25:36 +0000 | [diff] [blame] | 470 | if (MergedSize > MaxOffset) { |
| 471 | break; |
| 472 | } |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 473 | if (Padding) { |
| 474 | Tys.push_back(ArrayType::get(Int8Ty, Padding)); |
| 475 | Inits.push_back(ConstantAggregateZero::get(Tys.back())); |
| 476 | ++CurIdx; |
| 477 | } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 478 | Tys.push_back(Ty); |
| 479 | Inits.push_back(Globals[j]->getInitializer()); |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 480 | StructIdxs.push_back(CurIdx++); |
| 481 | |
| 482 | MaxAlign = std::max(MaxAlign, Align); |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 483 | |
| 484 | if (Globals[j]->hasExternalLinkage() && !HasExternal) { |
| 485 | HasExternal = true; |
Adrian Prantl | feb16c8 | 2016-11-11 22:09:25 +0000 | [diff] [blame] | 486 | FirstExternalName = Globals[j]->getName(); |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 487 | } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 488 | } |
| 489 | |
Haicheng Wu | b508121 | 2018-05-19 18:00:02 +0000 | [diff] [blame] | 490 | // Exit early if there is only one global to merge. |
| 491 | if (Tys.size() < 2) { |
| 492 | i = j; |
| 493 | continue; |
| 494 | } |
| 495 | |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 496 | // If merged variables doesn't have external linkage, we needn't to expose |
| 497 | // the symbol after merging. |
| 498 | GlobalValue::LinkageTypes Linkage = HasExternal |
| 499 | ? GlobalValue::ExternalLinkage |
| 500 | : GlobalValue::InternalLinkage; |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 501 | // Use a packed struct so we can control alignment. |
| 502 | StructType *MergedTy = StructType::get(M.getContext(), Tys, true); |
Chris Lattner | 252b491 | 2010-09-05 21:18:45 +0000 | [diff] [blame] | 503 | Constant *MergedInit = ConstantStruct::get(MergedTy, Inits); |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 504 | |
Adrian Prantl | cd5b334 | 2016-11-11 21:48:09 +0000 | [diff] [blame] | 505 | // On Darwin external linkage needs to be preserved, otherwise |
| 506 | // dsymutil cannot preserve the debug info for the merged |
| 507 | // variables. If they have external linkage, use the symbol name |
| 508 | // of the first variable merged as the suffix of global symbol |
| 509 | // name. This avoids a link-time naming conflict for the |
| 510 | // _MergedGlobals symbols. |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 511 | Twine MergedName = |
| 512 | (IsMachO && HasExternal) |
Adrian Prantl | feb16c8 | 2016-11-11 22:09:25 +0000 | [diff] [blame] | 513 | ? "_MergedGlobals_" + FirstExternalName |
Adrian Prantl | ec6378f | 2016-11-11 17:50:09 +0000 | [diff] [blame] | 514 | : "_MergedGlobals"; |
| 515 | auto MergedLinkage = IsMachO ? Linkage : GlobalValue::PrivateLinkage; |
| 516 | auto *MergedGV = new GlobalVariable( |
| 517 | M, MergedTy, isConst, MergedLinkage, MergedInit, MergedName, nullptr, |
| 518 | GlobalVariable::NotThreadLocal, AddrSpace); |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 519 | |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 520 | MergedGV->setAlignment(MaxAlign); |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 521 | MergedGV->setSection(Globals[i]->getSection()); |
Peter Collingbourne | 5420de3 | 2016-09-13 01:12:59 +0000 | [diff] [blame] | 522 | |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 523 | const StructLayout *MergedLayout = DL.getStructLayout(MergedTy); |
David Blaikie | 3beddee | 2015-09-14 20:29:26 +0000 | [diff] [blame] | 524 | for (ssize_t k = i, idx = 0; k != j; k = GlobalSet.find_next(k), ++idx) { |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 525 | GlobalValue::LinkageTypes Linkage = Globals[k]->getLinkage(); |
| 526 | std::string Name = Globals[k]->getName(); |
Martin Storsjo | 0d2725c | 2018-02-12 21:14:21 +0000 | [diff] [blame] | 527 | GlobalValue::DLLStorageClassTypes DLLStorage = |
| 528 | Globals[k]->getDLLStorageClass(); |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 529 | |
Peter Collingbourne | 5420de3 | 2016-09-13 01:12:59 +0000 | [diff] [blame] | 530 | // Copy metadata while adjusting any debug info metadata by the original |
| 531 | // global's offset within the merged global. |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 532 | MergedGV->copyMetadata(Globals[k], |
| 533 | MergedLayout->getElementOffset(StructIdxs[idx])); |
Peter Collingbourne | 5420de3 | 2016-09-13 01:12:59 +0000 | [diff] [blame] | 534 | |
Chris Lattner | 252b491 | 2010-09-05 21:18:45 +0000 | [diff] [blame] | 535 | Constant *Idx[2] = { |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 536 | ConstantInt::get(Int32Ty, 0), |
| 537 | ConstantInt::get(Int32Ty, StructIdxs[idx]), |
Chris Lattner | 252b491 | 2010-09-05 21:18:45 +0000 | [diff] [blame] | 538 | }; |
David Blaikie | 19443c1 | 2015-04-02 18:55:32 +0000 | [diff] [blame] | 539 | Constant *GEP = |
| 540 | ConstantExpr::getInBoundsGetElementPtr(MergedTy, MergedGV, Idx); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 541 | Globals[k]->replaceAllUsesWith(GEP); |
| 542 | Globals[k]->eraseFromParent(); |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 543 | |
John Brawn | 1a5ed7b | 2015-08-12 13:36:48 +0000 | [diff] [blame] | 544 | // When the linkage is not internal we must emit an alias for the original |
| 545 | // variable name as it may be accessed from another object. On non-Mach-O |
| 546 | // we can also emit an alias for internal linkage as it's safe to do so. |
| 547 | // It's not safe on Mach-O as the alias (and thus the portion of the |
| 548 | // MergedGlobals variable) may be dead stripped at link time. |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 549 | if (Linkage != GlobalValue::InternalLinkage || !IsMachO) { |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 550 | GlobalAlias *GA = GlobalAlias::create(Tys[StructIdxs[idx]], AddrSpace, |
| 551 | Linkage, Name, GEP, &M); |
Martin Storsjo | 0d2725c | 2018-02-12 21:14:21 +0000 | [diff] [blame] | 552 | GA->setDLLStorageClass(DLLStorage); |
John Brawn | 1a5ed7b | 2015-08-12 13:36:48 +0000 | [diff] [blame] | 553 | } |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 554 | |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 555 | NumMerged++; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 556 | } |
Haicheng Wu | b508121 | 2018-05-19 18:00:02 +0000 | [diff] [blame] | 557 | Changed = true; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 558 | i = j; |
| 559 | } |
| 560 | |
Haicheng Wu | b508121 | 2018-05-19 18:00:02 +0000 | [diff] [blame] | 561 | return Changed; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 562 | } |
| 563 | |
Eli Friedman | 261e479 | 2018-07-25 22:03:35 +0000 | [diff] [blame] | 564 | void GlobalMerge::collectUsedGlobalVariables(Module &M, StringRef Name) { |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 565 | // Extract global variables from llvm.used array |
Eli Friedman | 261e479 | 2018-07-25 22:03:35 +0000 | [diff] [blame] | 566 | const GlobalVariable *GV = M.getGlobalVariable(Name); |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 567 | if (!GV || !GV->hasInitializer()) return; |
| 568 | |
| 569 | // Should be an array of 'i8*'. |
Rafael Espindola | cde25b4 | 2013-04-22 14:58:02 +0000 | [diff] [blame] | 570 | const ConstantArray *InitList = cast<ConstantArray>(GV->getInitializer()); |
| 571 | |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 572 | for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) |
| 573 | if (const GlobalVariable *G = |
| 574 | dyn_cast<GlobalVariable>(InitList->getOperand(i)->stripPointerCasts())) |
| 575 | MustKeepGlobalVariables.insert(G); |
| 576 | } |
| 577 | |
| 578 | void GlobalMerge::setMustKeepGlobalVariables(Module &M) { |
Eli Friedman | 261e479 | 2018-07-25 22:03:35 +0000 | [diff] [blame] | 579 | collectUsedGlobalVariables(M, "llvm.used"); |
| 580 | collectUsedGlobalVariables(M, "llvm.compiler.used"); |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 581 | |
Reid Kleckner | 3fd59c9 | 2016-10-19 19:56:22 +0000 | [diff] [blame] | 582 | for (Function &F : M) { |
| 583 | for (BasicBlock &BB : F) { |
| 584 | Instruction *Pad = BB.getFirstNonPHI(); |
| 585 | if (!Pad->isEHPad()) |
| 586 | continue; |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 587 | |
Reid Kleckner | 3fd59c9 | 2016-10-19 19:56:22 +0000 | [diff] [blame] | 588 | // Keep globals used by landingpads and catchpads. |
| 589 | for (const Use &U : Pad->operands()) { |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 590 | if (const GlobalVariable *GV = |
Reid Kleckner | 3fd59c9 | 2016-10-19 19:56:22 +0000 | [diff] [blame] | 591 | dyn_cast<GlobalVariable>(U->stripPointerCasts())) |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 592 | MustKeepGlobalVariables.insert(GV); |
Reid Kleckner | 3fd59c9 | 2016-10-19 19:56:22 +0000 | [diff] [blame] | 593 | } |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 594 | } |
| 595 | } |
| 596 | } |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 597 | |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 598 | bool GlobalMerge::doInitialization(Module &M) { |
Tim Northover | 4bcb985 | 2014-02-18 11:17:29 +0000 | [diff] [blame] | 599 | if (!EnableGlobalMerge) |
| 600 | return false; |
| 601 | |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 602 | IsMachO = Triple(M.getTargetTriple()).isOSBinFormatMachO(); |
| 603 | |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 604 | auto &DL = M.getDataLayout(); |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 605 | DenseMap<std::pair<unsigned, StringRef>, SmallVector<GlobalVariable *, 16>> |
| 606 | Globals, ConstGlobals, BSSGlobals; |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 607 | bool Changed = false; |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 608 | setMustKeepGlobalVariables(M); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 609 | |
| 610 | // Grab all non-const globals. |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 611 | for (auto &GV : M.globals()) { |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 612 | // Merge is safe for "normal" internal or external globals only |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 613 | if (GV.isDeclaration() || GV.isThreadLocal() || GV.hasImplicitSection()) |
Jiangning Liu | f847ccb | 2014-06-11 06:44:53 +0000 | [diff] [blame] | 614 | continue; |
| 615 | |
John Brawn | 0f1cadd | 2017-06-02 10:24:14 +0000 | [diff] [blame] | 616 | // It's not safe to merge globals that may be preempted |
| 617 | if (TM && !TM->shouldAssumeDSOLocal(M, &GV)) |
| 618 | continue; |
| 619 | |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 620 | if (!(MergeExternalGlobals && GV.hasExternalLinkage()) && |
| 621 | !GV.hasInternalLinkage()) |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 622 | continue; |
| 623 | |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 624 | PointerType *PT = dyn_cast<PointerType>(GV.getType()); |
Silviu Baranga | e971659 | 2013-01-07 12:31:25 +0000 | [diff] [blame] | 625 | assert(PT && "Global variable is not a pointer!"); |
| 626 | |
| 627 | unsigned AddressSpace = PT->getAddressSpace(); |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 628 | StringRef Section = GV.getSection(); |
Silviu Baranga | e971659 | 2013-01-07 12:31:25 +0000 | [diff] [blame] | 629 | |
Anton Korobeynikov | b5a0ef9 | 2010-07-26 18:45:39 +0000 | [diff] [blame] | 630 | // Ignore all 'special' globals. |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 631 | if (GV.getName().startswith("llvm.") || |
| 632 | GV.getName().startswith(".llvm.")) |
Anton Korobeynikov | b5a0ef9 | 2010-07-26 18:45:39 +0000 | [diff] [blame] | 633 | continue; |
| 634 | |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 635 | // Ignore all "required" globals: |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 636 | if (isMustKeepGlobalVariable(&GV)) |
Quentin Colombet | e572809 | 2013-03-18 22:30:07 +0000 | [diff] [blame] | 637 | continue; |
| 638 | |
Eli Friedman | fab866f | 2018-07-25 20:58:01 +0000 | [diff] [blame] | 639 | Type *Ty = GV.getValueType(); |
Mehdi Amini | a777964 | 2015-07-07 18:49:25 +0000 | [diff] [blame] | 640 | if (DL.getTypeAllocSize(Ty) < MaxOffset) { |
Peter Collingbourne | 28f667b | 2016-05-19 04:38:56 +0000 | [diff] [blame] | 641 | if (TM && |
Huihui Zhang | 6f7036e | 2018-08-30 00:49:50 +0000 | [diff] [blame] | 642 | TargetLoweringObjectFile::getKindForGlobal(&GV, *TM).isBSS()) |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 643 | BSSGlobals[{AddressSpace, Section}].push_back(&GV); |
Duncan P. N. Exon Smith | 8d36360 | 2015-10-09 18:57:47 +0000 | [diff] [blame] | 644 | else if (GV.isConstant()) |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 645 | ConstGlobals[{AddressSpace, Section}].push_back(&GV); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 646 | else |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 647 | Globals[{AddressSpace, Section}].push_back(&GV); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 648 | } |
| 649 | } |
| 650 | |
David Blaikie | f47cf24 | 2015-08-21 22:19:06 +0000 | [diff] [blame] | 651 | for (auto &P : Globals) |
| 652 | if (P.second.size() > 1) |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 653 | Changed |= doMerge(P.second, M, false, P.first.first); |
Silviu Baranga | e971659 | 2013-01-07 12:31:25 +0000 | [diff] [blame] | 654 | |
David Blaikie | f47cf24 | 2015-08-21 22:19:06 +0000 | [diff] [blame] | 655 | for (auto &P : BSSGlobals) |
| 656 | if (P.second.size() > 1) |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 657 | Changed |= doMerge(P.second, M, false, P.first.first); |
Bob Wilson | 0564609 | 2010-11-17 21:25:39 +0000 | [diff] [blame] | 658 | |
David Blaikie | 47c36ab | 2015-08-25 17:01:36 +0000 | [diff] [blame] | 659 | if (EnableGlobalMergeOnConst) |
| 660 | for (auto &P : ConstGlobals) |
| 661 | if (P.second.size() > 1) |
Eli Friedman | f4a5ef1 | 2018-08-02 23:54:16 +0000 | [diff] [blame] | 662 | Changed |= doMerge(P.second, M, true, P.first.first); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 663 | |
| 664 | return Changed; |
| 665 | } |
| 666 | |
Devang Patel | 827454e | 2011-10-17 17:17:43 +0000 | [diff] [blame] | 667 | bool GlobalMerge::runOnFunction(Function &F) { |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 668 | return false; |
| 669 | } |
| 670 | |
Quentin Colombet | 9deb917 | 2013-03-19 21:46:49 +0000 | [diff] [blame] | 671 | bool GlobalMerge::doFinalization(Module &M) { |
| 672 | MustKeepGlobalVariables.clear(); |
| 673 | return false; |
| 674 | } |
| 675 | |
Ahmed Bougacha | 0d9335e | 2015-06-04 20:39:23 +0000 | [diff] [blame] | 676 | Pass *llvm::createGlobalMergePass(const TargetMachine *TM, unsigned Offset, |
John Brawn | 583fe8f | 2015-08-03 12:08:41 +0000 | [diff] [blame] | 677 | bool OnlyOptimizeForSize, |
| 678 | bool MergeExternalByDefault) { |
| 679 | bool MergeExternal = (EnableGlobalMergeOnExternal == cl::BOU_UNSET) ? |
| 680 | MergeExternalByDefault : (EnableGlobalMergeOnExternal == cl::BOU_TRUE); |
| 681 | return new GlobalMerge(TM, Offset, OnlyOptimizeForSize, MergeExternal); |
Anton Korobeynikov | cec36f4 | 2010-07-24 21:52:08 +0000 | [diff] [blame] | 682 | } |