Ninja
deps_log_test.cc
Go to the documentation of this file.
1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "deps_log.h"
16 
17 #include "graph.h"
18 #include "util.h"
19 #include "test.h"
20 
21 namespace {
22 
23 const char kTestFilename[] = "DepsLogTest-tempfile";
24 
25 struct DepsLogTest : public testing::Test {
26  virtual void SetUp() {
27  // In case a crashing test left a stale file behind.
28  unlink(kTestFilename);
29  }
30  virtual void TearDown() {
31  unlink(kTestFilename);
32  }
33 };
34 
35 TEST_F(DepsLogTest, WriteRead) {
36  State state1;
37  DepsLog log1;
38  string err;
39  EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
40  ASSERT_EQ("", err);
41 
42  {
43  vector<Node*> deps;
44  deps.push_back(state1.GetNode("foo.h"));
45  deps.push_back(state1.GetNode("bar.h"));
46  log1.RecordDeps(state1.GetNode("out.o"), 1, deps);
47 
48  deps.clear();
49  deps.push_back(state1.GetNode("foo.h"));
50  deps.push_back(state1.GetNode("bar2.h"));
51  log1.RecordDeps(state1.GetNode("out2.o"), 2, deps);
52 
53  DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o"));
54  ASSERT_TRUE(log_deps);
55  ASSERT_EQ(1, log_deps->mtime);
56  ASSERT_EQ(2, log_deps->node_count);
57  ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
58  ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
59  }
60 
61  log1.Close();
62 
63  State state2;
64  DepsLog log2;
65  EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
66  ASSERT_EQ("", err);
67 
68  ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
69  for (int i = 0; i < (int)log1.nodes().size(); ++i) {
70  Node* node1 = log1.nodes()[i];
71  Node* node2 = log2.nodes()[i];
72  ASSERT_EQ(i, node1->id());
73  ASSERT_EQ(node1->id(), node2->id());
74  }
75 
76  // Spot-check the entries in log2.
77  DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o"));
78  ASSERT_TRUE(log_deps);
79  ASSERT_EQ(2, log_deps->mtime);
80  ASSERT_EQ(2, log_deps->node_count);
81  ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
82  ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
83 }
84 
85 TEST_F(DepsLogTest, LotsOfDeps) {
86  const int kNumDeps = 100000; // More than 64k.
87 
88  State state1;
89  DepsLog log1;
90  string err;
91  EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
92  ASSERT_EQ("", err);
93 
94  {
95  vector<Node*> deps;
96  for (int i = 0; i < kNumDeps; ++i) {
97  char buf[32];
98  sprintf(buf, "file%d.h", i);
99  deps.push_back(state1.GetNode(buf));
100  }
101  log1.RecordDeps(state1.GetNode("out.o"), 1, deps);
102 
103  DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o"));
104  ASSERT_EQ(kNumDeps, log_deps->node_count);
105  }
106 
107  log1.Close();
108 
109  State state2;
110  DepsLog log2;
111  EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
112  ASSERT_EQ("", err);
113 
114  DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o"));
115  ASSERT_EQ(kNumDeps, log_deps->node_count);
116 }
117 
118 // Verify that adding the same deps twice doesn't grow the file.
119 TEST_F(DepsLogTest, DoubleEntry) {
120  // Write some deps to the file and grab its size.
121  int file_size;
122  {
123  State state;
124  DepsLog log;
125  string err;
126  EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
127  ASSERT_EQ("", err);
128 
129  vector<Node*> deps;
130  deps.push_back(state.GetNode("foo.h"));
131  deps.push_back(state.GetNode("bar.h"));
132  log.RecordDeps(state.GetNode("out.o"), 1, deps);
133  log.Close();
134 
135  struct stat st;
136  ASSERT_EQ(0, stat(kTestFilename, &st));
137  file_size = (int)st.st_size;
138  ASSERT_GT(file_size, 0);
139  }
140 
141  // Now reload the file, and readd the same deps.
142  {
143  State state;
144  DepsLog log;
145  string err;
146  EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
147 
148  EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
149  ASSERT_EQ("", err);
150 
151  vector<Node*> deps;
152  deps.push_back(state.GetNode("foo.h"));
153  deps.push_back(state.GetNode("bar.h"));
154  log.RecordDeps(state.GetNode("out.o"), 1, deps);
155  log.Close();
156 
157  struct stat st;
158  ASSERT_EQ(0, stat(kTestFilename, &st));
159  int file_size_2 = (int)st.st_size;
160  ASSERT_EQ(file_size, file_size_2);
161  }
162 }
163 
164 // Verify that adding the new deps works and can be compacted away.
165 TEST_F(DepsLogTest, Recompact) {
166  const char kManifest[] =
167 "rule cc\n"
168 " command = cc\n"
169 " deps = gcc\n"
170 "build out.o: cc\n"
171 "build other_out.o: cc\n";
172 
173  // Write some deps to the file and grab its size.
174  int file_size;
175  {
176  State state;
177  ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
178  DepsLog log;
179  string err;
180  ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
181  ASSERT_EQ("", err);
182 
183  vector<Node*> deps;
184  deps.push_back(state.GetNode("foo.h"));
185  deps.push_back(state.GetNode("bar.h"));
186  log.RecordDeps(state.GetNode("out.o"), 1, deps);
187 
188  deps.clear();
189  deps.push_back(state.GetNode("foo.h"));
190  deps.push_back(state.GetNode("baz.h"));
191  log.RecordDeps(state.GetNode("other_out.o"), 1, deps);
192 
193  log.Close();
194 
195  struct stat st;
196  ASSERT_EQ(0, stat(kTestFilename, &st));
197  file_size = (int)st.st_size;
198  ASSERT_GT(file_size, 0);
199  }
200 
201  // Now reload the file, and add slighly different deps.
202  int file_size_2;
203  {
204  State state;
205  ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
206  DepsLog log;
207  string err;
208  ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
209 
210  ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
211  ASSERT_EQ("", err);
212 
213  vector<Node*> deps;
214  deps.push_back(state.GetNode("foo.h"));
215  log.RecordDeps(state.GetNode("out.o"), 1, deps);
216  log.Close();
217 
218  struct stat st;
219  ASSERT_EQ(0, stat(kTestFilename, &st));
220  file_size_2 = (int)st.st_size;
221  // The file should grow to record the new deps.
222  ASSERT_GT(file_size_2, file_size);
223  }
224 
225  // Now reload the file, verify the new deps have replaced the old, then
226  // recompact.
227  int file_size_3;
228  {
229  State state;
230  ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
231  DepsLog log;
232  string err;
233  ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
234 
235  Node* out = state.GetNode("out.o");
236  DepsLog::Deps* deps = log.GetDeps(out);
237  ASSERT_TRUE(deps);
238  ASSERT_EQ(1, deps->mtime);
239  ASSERT_EQ(1, deps->node_count);
240  ASSERT_EQ("foo.h", deps->nodes[0]->path());
241 
242  Node* other_out = state.GetNode("other_out.o");
243  deps = log.GetDeps(other_out);
244  ASSERT_TRUE(deps);
245  ASSERT_EQ(1, deps->mtime);
246  ASSERT_EQ(2, deps->node_count);
247  ASSERT_EQ("foo.h", deps->nodes[0]->path());
248  ASSERT_EQ("baz.h", deps->nodes[1]->path());
249 
250  ASSERT_TRUE(log.Recompact(kTestFilename, &err));
251 
252  // The in-memory deps graph should still be valid after recompaction.
253  deps = log.GetDeps(out);
254  ASSERT_TRUE(deps);
255  ASSERT_EQ(1, deps->mtime);
256  ASSERT_EQ(1, deps->node_count);
257  ASSERT_EQ("foo.h", deps->nodes[0]->path());
258  ASSERT_EQ(out, log.nodes()[out->id()]);
259 
260  deps = log.GetDeps(other_out);
261  ASSERT_TRUE(deps);
262  ASSERT_EQ(1, deps->mtime);
263  ASSERT_EQ(2, deps->node_count);
264  ASSERT_EQ("foo.h", deps->nodes[0]->path());
265  ASSERT_EQ("baz.h", deps->nodes[1]->path());
266  ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
267 
268  // The file should have shrunk a bit for the smaller deps.
269  struct stat st;
270  ASSERT_EQ(0, stat(kTestFilename, &st));
271  file_size_3 = (int)st.st_size;
272  ASSERT_LT(file_size_3, file_size_2);
273  }
274 
275  // Now reload the file and recompact with an empty manifest. The previous
276  // entries should be removed.
277  {
278  State state;
279  // Intentionally not parsing kManifest here.
280  DepsLog log;
281  string err;
282  ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
283 
284  Node* out = state.GetNode("out.o");
285  DepsLog::Deps* deps = log.GetDeps(out);
286  ASSERT_TRUE(deps);
287  ASSERT_EQ(1, deps->mtime);
288  ASSERT_EQ(1, deps->node_count);
289  ASSERT_EQ("foo.h", deps->nodes[0]->path());
290 
291  Node* other_out = state.GetNode("other_out.o");
292  deps = log.GetDeps(other_out);
293  ASSERT_TRUE(deps);
294  ASSERT_EQ(1, deps->mtime);
295  ASSERT_EQ(2, deps->node_count);
296  ASSERT_EQ("foo.h", deps->nodes[0]->path());
297  ASSERT_EQ("baz.h", deps->nodes[1]->path());
298 
299  ASSERT_TRUE(log.Recompact(kTestFilename, &err));
300 
301  // The previous entries should have been removed.
302  deps = log.GetDeps(out);
303  ASSERT_FALSE(deps);
304 
305  deps = log.GetDeps(other_out);
306  ASSERT_FALSE(deps);
307 
308  // The .h files pulled in via deps should no longer have ids either.
309  ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
310  ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
311 
312  // The file should have shrunk more.
313  struct stat st;
314  ASSERT_EQ(0, stat(kTestFilename, &st));
315  int file_size_4 = (int)st.st_size;
316  ASSERT_LT(file_size_4, file_size_3);
317  }
318 }
319 
320 // Verify that invalid file headers cause a new build.
321 TEST_F(DepsLogTest, InvalidHeader) {
322  const char *kInvalidHeaders[] = {
323  "", // Empty file.
324  "# ninjad", // Truncated first line.
325  "# ninjadeps\n", // No version int.
326  "# ninjadeps\n\001\002", // Truncated version int.
327  "# ninjadeps\n\001\002\003\004" // Invalid version int.
328  };
329  for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
330  ++i) {
331  FILE* deps_log = fopen(kTestFilename, "wb");
332  ASSERT_TRUE(deps_log != NULL);
333  ASSERT_EQ(
334  strlen(kInvalidHeaders[i]),
335  fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
336  ASSERT_EQ(0 ,fclose(deps_log));
337 
338  string err;
339  DepsLog log;
340  State state;
341  ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
342  EXPECT_EQ("bad deps log signature or version; starting over", err);
343  }
344 }
345 
346 // Simulate what happens when loading a truncated log file.
347 TEST_F(DepsLogTest, Truncated) {
348  // Create a file with some entries.
349  {
350  State state;
351  DepsLog log;
352  string err;
353  EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
354  ASSERT_EQ("", err);
355 
356  vector<Node*> deps;
357  deps.push_back(state.GetNode("foo.h"));
358  deps.push_back(state.GetNode("bar.h"));
359  log.RecordDeps(state.GetNode("out.o"), 1, deps);
360 
361  deps.clear();
362  deps.push_back(state.GetNode("foo.h"));
363  deps.push_back(state.GetNode("bar2.h"));
364  log.RecordDeps(state.GetNode("out2.o"), 2, deps);
365 
366  log.Close();
367  }
368 
369  // Get the file size.
370  struct stat st;
371  ASSERT_EQ(0, stat(kTestFilename, &st));
372 
373  // Try reloading at truncated sizes.
374  // Track how many nodes/deps were found; they should decrease with
375  // smaller sizes.
376  int node_count = 5;
377  int deps_count = 2;
378  for (int size = (int)st.st_size; size > 0; --size) {
379  string err;
380  ASSERT_TRUE(Truncate(kTestFilename, size, &err));
381 
382  State state;
383  DepsLog log;
384  EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
385  if (!err.empty()) {
386  // At some point the log will be so short as to be unparseable.
387  break;
388  }
389 
390  ASSERT_GE(node_count, (int)log.nodes().size());
391  node_count = log.nodes().size();
392 
393  // Count how many non-NULL deps entries there are.
394  int new_deps_count = 0;
395  for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
396  i != log.deps().end(); ++i) {
397  if (*i)
398  ++new_deps_count;
399  }
400  ASSERT_GE(deps_count, new_deps_count);
401  deps_count = new_deps_count;
402  }
403 }
404 
405 // Run the truncation-recovery logic.
406 TEST_F(DepsLogTest, TruncatedRecovery) {
407  // Create a file with some entries.
408  {
409  State state;
410  DepsLog log;
411  string err;
412  EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
413  ASSERT_EQ("", err);
414 
415  vector<Node*> deps;
416  deps.push_back(state.GetNode("foo.h"));
417  deps.push_back(state.GetNode("bar.h"));
418  log.RecordDeps(state.GetNode("out.o"), 1, deps);
419 
420  deps.clear();
421  deps.push_back(state.GetNode("foo.h"));
422  deps.push_back(state.GetNode("bar2.h"));
423  log.RecordDeps(state.GetNode("out2.o"), 2, deps);
424 
425  log.Close();
426  }
427 
428  // Shorten the file, corrupting the last record.
429  struct stat st;
430  ASSERT_EQ(0, stat(kTestFilename, &st));
431  string err;
432  ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
433 
434  // Load the file again, add an entry.
435  {
436  State state;
437  DepsLog log;
438  string err;
439  EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
440  ASSERT_EQ("premature end of file; recovering", err);
441  err.clear();
442 
443  // The truncated entry should've been discarded.
444  EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o")));
445 
446  EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
447  ASSERT_EQ("", err);
448 
449  // Add a new entry.
450  vector<Node*> deps;
451  deps.push_back(state.GetNode("foo.h"));
452  deps.push_back(state.GetNode("bar2.h"));
453  log.RecordDeps(state.GetNode("out2.o"), 3, deps);
454 
455  log.Close();
456  }
457 
458  // Load the file a third time to verify appending after a mangled
459  // entry doesn't break things.
460  {
461  State state;
462  DepsLog log;
463  string err;
464  EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
465 
466  // The truncated entry should exist.
467  DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o"));
468  ASSERT_TRUE(deps);
469  }
470 }
471 
472 } // anonymous namespace
Node * GetNode(StringPiece path)
Definition: state.cc:114
Information about a node in the dependency graph: the file, whether it's dirty, mtime, etc.
Definition: graph.h:35
Node ** nodes
Definition: deps_log.h:83
void AssertParse(State *state, const char *input)
Definition: test.cc:90
As build commands run they can output extra dependency information (e.g.
Definition: deps_log.h:66
bool OpenForWrite(const string &path, string *err)
Definition: deps_log.cc:43
TEST_F(PlanTest, Basic)
Definition: build_test.cc:51
Deps * GetDeps(Node *node)
Definition: deps_log.cc:295
const char kTestFilename[]
int node_count
Definition: deps_log.h:82
bool Load(const string &path, State *state, string *err)
Definition: deps_log.cc:164
bool Recompact(const string &path, string *err)
Rewrite the known log entries, throwing away old data.
Definition: deps_log.cc:303
const string & path() const
Definition: graph.h:73
void Close()
Definition: deps_log.cc:158
int id() const
Definition: graph.h:83
bool Truncate(const string &path, size_t size, string *err)
Truncates a file to the given size.
Definition: util.cc:447
const vector< Deps * > & deps() const
Definition: deps_log.h:101
Global state (file status, loaded rules) for a single run.
Definition: state.h:83
const vector< Node * > & nodes() const
Used for tests.
Definition: deps_log.h:100
bool RecordDeps(Node *node, TimeStamp mtime, const vector< Node * > &nodes)
Definition: deps_log.cc:80
Node * LookupNode(StringPiece path) const
Definition: state.cc:123