xref: /glogg/tests/testlogfiltereddata.cpp (revision 6e2e573c451ec24d720c967fab68538eaa3f3ab5) 
1 /*
2  * Copyright (C) 2009, 2010, 2011 Nicolas Bonnefon and other contributors
3  *
4  * This file is part of glogg.
5  *
6  * glogg is free software: you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation, either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * glogg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with glogg.  If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include <QSignalSpy>
21 #include <QMutexLocker>
22 #include <QFile>
23 
24 #include "testlogfiltereddata.h"
25 #include "logdata.h"
26 #include "logfiltereddata.h"
27 
28 #if !defined( TMPDIR )
29 #define TMPDIR "/tmp"
30 #endif
31 
32 static const qint64 ML_NB_LINES = 15000LL;
33 static const char* ml_format="LOGDATA is a part of glogg, we are going to test it thoroughly, this is line\t\t%06d\n";
34 static const int ML_VISIBLE_LINE_LENGTH = (76+8+4+6); // Without the final '\n' !
35 
36 static const qint64 SL_NB_LINES = 2000LL;
37 static const char* sl_format="LOGDATA is a part of glogg, we are going to test it thoroughly, this is line %06d\n";
38 
39 static const char* partial_line_begin = "123... beginning of line.";
40 static const char* partial_line_end = " end of line 123.\n";
41 
42 static const char* partial_nonmatching_line_begin = "Beginning of line.";
43 
44 TestLogFilteredData::TestLogFilteredData() : QObject(),
45     loadingFinishedMutex_(),
46     searchProgressedMutex_(),
47     loadingFinishedCondition_(),
48     searchProgressedCondition_()
49 {
50     loadingFinished_received_  = false;
51     loadingFinished_read_      = false;
52     searchProgressed_received_ = false;
53     searchProgressed_read_     = false;
54 }
55 
56 void TestLogFilteredData::initTestCase()
57 {
58     QVERIFY( generateDataFiles() );
59 }
60 
61 void TestLogFilteredData::simpleSearch()
62 {
63     logData_ = new LogData();
64 
65     // Register for notification file is loaded
66     connect( logData_, SIGNAL( loadingFinished( bool ) ),
67             this, SLOT( loadingFinished() ) );
68 
69     filteredData_ = logData_->getNewFilteredData();
70     connect( filteredData_, SIGNAL( searchProgressed( int, int ) ),
71             this, SLOT( searchProgressed( int, int ) ) );
72 
73     QFuture<void> future = QtConcurrent::run(this, &TestLogFilteredData::simpleSearchTest);
74 
75     QApplication::exec();
76 
77     disconnect( filteredData_, 0 );
78     disconnect( logData_, 0 );
79 
80     delete filteredData_;
81     delete logData_;
82 }
83 
84 void TestLogFilteredData::simpleSearchTest()
85 {
86     // First load the tests file
87     logData_->attachFile( TMPDIR "/mediumlog.txt" );
88     // Wait for the loading to be done
89     waitLoadingFinished();
90     QCOMPARE( logData_->getNbLine(), ML_NB_LINES );
91     signalLoadingFinishedRead();
92 
93     // Now perform a simple search
94     qint64 matches[] = { 0, 15, 20, 135 };
95     QBENCHMARK {
96         // Start the search
97         filteredData_->runSearch( QRegExp( "123" ) );
98 
99         // And check we receive data in 4 chunks (the first being empty)
100         for ( int i = 0; i < 4; i++ ) {
101             std::pair<int,int> progress = waitSearchProgressed();
102             // FIXME: The test for this is unfortunately not reliable
103             // (race conditions)
104             // QCOMPARE( (qint64) progress.first, matches[i] );
105             signalSearchProgressedRead();
106         }
107     }
108 
109     QCOMPARE( filteredData_->getNbLine(), matches[3] );
110     // Check the search
111     QCOMPARE( filteredData_->isLineInMatchingList( 123 ), true );
112     QCOMPARE( filteredData_->isLineInMatchingList( 124 ), false );
113     QCOMPARE( filteredData_->getMaxLength(), ML_VISIBLE_LINE_LENGTH );
114     QCOMPARE( filteredData_->getLineLength( 12 ), ML_VISIBLE_LINE_LENGTH );
115     QCOMPARE( filteredData_->getNbLine(), 135LL );
116     // Line beyond limit
117     QCOMPARE( filteredData_->isLineInMatchingList( 60000 ), false );
118     QCOMPARE( filteredData_->getMatchingLineNumber( 0 ), 123LL );
119 
120     // Now let's try interrupting a search
121     filteredData_->runSearch( QRegExp( "123" ) );
122     // ... wait for two chunks.
123     waitSearchProgressed();
124     signalSearchProgressedRead();
125     // and interrupt!
126     filteredData_->interruptSearch();
127 
128     {
129         std::pair<int,int> progress;
130         do {
131             progress = waitSearchProgressed();
132             signalSearchProgressedRead();
133         } while ( progress.second < 100 );
134 
135         // (because there is no guarantee when the search is
136         // interrupted, we are not sure how many chunk of result
137         // we will get.)
138     }
139 
140     QApplication::quit();
141 }
142 
143 void TestLogFilteredData::multipleSearch()
144 {
145     logData_ = new LogData();
146 
147     // Register for notification file is loaded
148     connect( logData_, SIGNAL( loadingFinished( bool ) ),
149             this, SLOT( loadingFinished() ) );
150 
151     filteredData_ = logData_->getNewFilteredData();
152     connect( filteredData_, SIGNAL( searchProgressed( int, int ) ),
153             this, SLOT( searchProgressed( int, int ) ) );
154 
155     QFuture<void> future = QtConcurrent::run(this, &TestLogFilteredData::multipleSearchTest);
156 
157     QApplication::exec();
158 
159     disconnect( filteredData_, 0 );
160     disconnect( logData_, 0 );
161 
162     delete filteredData_;
163     delete logData_;
164 }
165 
166 void TestLogFilteredData::multipleSearchTest()
167 {
168     // First load the tests file
169     logData_->attachFile( TMPDIR "/smalllog.txt" );
170     // Wait for the loading to be done
171     waitLoadingFinished();
172     QCOMPARE( logData_->getNbLine(), SL_NB_LINES );
173     signalLoadingFinishedRead();
174 
175     // Performs two searches in a row
176     // Start the search, and immediately another one
177     // (the second call should block until the first search is done)
178     filteredData_->runSearch( QRegExp( "1234" ) );
179     filteredData_->runSearch( QRegExp( "123" ) );
180 
181     for ( int i = 0; i < 3; i++ ) {
182         waitSearchProgressed();
183         signalSearchProgressedRead();
184     }
185 
186     // We should have the result for the 2nd search after the last chunk
187     waitSearchProgressed();
188     QCOMPARE( filteredData_->getNbLine(), 12LL );
189     signalSearchProgressedRead();
190 
191     // Now a tricky one: we run a search and immediately attach a new file
192     /* FIXME: sometimes we receive loadingFinished before searchProgressed
193      * -> deadlock in the test.
194     filteredData_->runSearch( QRegExp( "123" ) );
195     waitSearchProgressed();
196     signalSearchProgressedRead();
197     logData_->attachFile( TMPDIR "/mediumlog.txt" );
198 
199     // We don't expect meaningful results but it should not crash!
200     for ( int i = 0; i < 1; i++ ) {
201         waitSearchProgressed();
202         signalSearchProgressedRead();
203     }
204     */
205 
206     sleep(10);
207 
208     QApplication::quit();
209 }
210 
211 void TestLogFilteredData::updateSearch()
212 {
213     logData_ = new LogData();
214 
215     // Register for notification file is loaded
216     connect( logData_, SIGNAL( loadingFinished( bool ) ),
217             this, SLOT( loadingFinished() ) );
218 
219     filteredData_ = logData_->getNewFilteredData();
220     connect( filteredData_, SIGNAL( searchProgressed( int, int ) ),
221             this, SLOT( searchProgressed( int, int ) ) );
222 
223     QFuture<void> future = QtConcurrent::run(this, &TestLogFilteredData::updateSearchTest);
224 
225     QApplication::exec();
226 
227     disconnect( filteredData_, 0 );
228     disconnect( logData_, 0 );
229 
230     delete filteredData_;
231     delete logData_;
232 }
233 
234 void TestLogFilteredData::updateSearchTest()
235 {
236     // First load the tests file
237     logData_->attachFile( TMPDIR "/smalllog.txt" );
238     // Wait for the loading to be done
239     waitLoadingFinished();
240     QCOMPARE( logData_->getNbLine(), SL_NB_LINES );
241     signalLoadingFinishedRead();
242 
243     // Perform a first search
244     filteredData_->runSearch( QRegExp( "123" ) );
245 
246     for ( int i = 0; i < 2; i++ ) {
247         waitSearchProgressed();
248         signalSearchProgressedRead();
249     }
250 
251     // Check the result
252     QCOMPARE( filteredData_->getNbLine(), 12LL );
253 
254     sleep(1);
255 
256     QWARN("Starting stage 2");
257 
258     // Add some data to the file
259     char newLine[90];
260     QFile file( TMPDIR "/smalllog.txt" );
261     if ( file.open( QIODevice::Append ) ) {
262         for (int i = 0; i < 3000; i++) {
263             snprintf(newLine, 89, sl_format, i);
264             file.write( newLine, qstrlen(newLine) );
265         }
266         // To test the edge case when the final line is not complete and matching
267         file.write( partial_line_begin, qstrlen( partial_line_begin ) );
268     }
269     file.close();
270 
271     // Let the system do the update (there might be several ones)
272     do {
273         waitLoadingFinished();
274         signalLoadingFinishedRead();
275     } while ( logData_->getNbLine() < 5001LL );
276 
277     sleep(1);
278 
279     // Start an update search
280     filteredData_->updateSearch();
281 
282     for ( int i = 0; i < 2; i++ ) {
283         waitSearchProgressed();
284         signalSearchProgressedRead();
285     }
286 
287     // Check the result
288     QCOMPARE( logData_->getNbLine(), 5001LL );
289     QCOMPARE( filteredData_->getNbLine(), 26LL );
290 
291     QWARN("Starting stage 3");
292 
293     // Add a couple more lines, including the end of the unfinished one.
294     if ( file.open( QIODevice::Append ) ) {
295         file.write( partial_line_end, qstrlen( partial_line_end ) );
296         for (int i = 0; i < 20; i++) {
297             snprintf(newLine, 89, sl_format, i);
298             file.write( newLine, qstrlen(newLine) );
299         }
300         // To test the edge case when the final line is not complete and not matching
301         file.write( partial_nonmatching_line_begin,
302                 qstrlen( partial_nonmatching_line_begin ) );
303     }
304     file.close();
305 
306     // Let the system do the update
307     waitLoadingFinished();
308     signalLoadingFinishedRead();
309 
310     // Start an update search
311     filteredData_->updateSearch();
312 
313     for ( int i = 0; i < 2; i++ ) {
314         waitSearchProgressed();
315         signalSearchProgressedRead();
316     }
317 
318     // Check the result
319     QCOMPARE( logData_->getNbLine(), 5022LL );
320     QCOMPARE( filteredData_->getNbLine(), 26LL );
321 
322     QWARN("Starting stage 4");
323 
324     // Now test the case where a match is found at the end of an updated line.
325     if ( file.open( QIODevice::Append ) ) {
326         file.write( partial_line_end, qstrlen( partial_line_end ) );
327         for (int i = 0; i < 20; i++) {
328             snprintf(newLine, 89, sl_format, i);
329             file.write( newLine, qstrlen(newLine) );
330         }
331     }
332     file.close();
333 
334     // Let the system do the update
335     waitLoadingFinished();
336     signalLoadingFinishedRead();
337 
338     // Start an update search
339     filteredData_->updateSearch();
340 
341     for ( int i = 0; i < 2; i++ ) {
342         waitSearchProgressed();
343         signalSearchProgressedRead();
344     }
345 
346     // Check the result
347     QCOMPARE( logData_->getNbLine(), 5042LL );
348     QCOMPARE( filteredData_->getNbLine(), 27LL );
349 
350     QApplication::quit();
351 }
352 
353 void TestLogFilteredData::marks()
354 {
355     logData_ = new LogData();
356 
357     // Register for notification file is loaded
358     connect( logData_, SIGNAL( loadingFinished( bool ) ),
359             this, SLOT( loadingFinished() ) );
360 
361     filteredData_ = logData_->getNewFilteredData();
362     connect( filteredData_, SIGNAL( searchProgressed( int, int ) ),
363             this, SLOT( searchProgressed( int, int ) ) );
364 
365     QFuture<void> future = QtConcurrent::run(this, &TestLogFilteredData::marksTest);
366 
367     QApplication::exec();
368 
369     disconnect( filteredData_, 0 );
370     disconnect( logData_, 0 );
371 
372     delete filteredData_;
373     delete logData_;
374 }
375 
376 void TestLogFilteredData::marksTest()
377 {
378     // First load the tests file
379     logData_->attachFile( TMPDIR "/smalllog.txt" );
380     // Wait for the loading to be done
381     waitLoadingFinished();
382     QCOMPARE( logData_->getNbLine(), SL_NB_LINES );
383     signalLoadingFinishedRead();
384 
385     // First check no line is marked
386     for ( int i = 0; i < SL_NB_LINES; i++ )
387         QVERIFY( filteredData_->isLineMarked( i ) == false );
388 
389     // Try to create some "out of limit" marks
390     filteredData_->addMark( -10 );
391     filteredData_->addMark( SL_NB_LINES + 25 );
392 
393     // Check no line is marked still
394     for ( int i = 0; i < SL_NB_LINES; i++ )
395         QVERIFY( filteredData_->isLineMarked( i ) == false );
396 
397     // Create a couple of unnamed marks
398     filteredData_->addMark( 10 );
399     filteredData_->addMark( 44 );  // This one will also be a match
400     filteredData_->addMark( 25 );
401 
402     // Check they are marked
403     QVERIFY( filteredData_->isLineMarked( 10 ) );
404     QVERIFY( filteredData_->isLineMarked( 25 ) );
405     QVERIFY( filteredData_->isLineMarked( 44 ) );
406 
407     // But others are not
408     QVERIFY( filteredData_->isLineMarked( 15 ) == false );
409     QVERIFY( filteredData_->isLineMarked( 20 ) == false );
410 
411     QCOMPARE( filteredData_->getNbLine(), 3LL );
412 
413     // Performs a search
414     QSignalSpy progressSpy( filteredData_,
415             SIGNAL( searchProgressed( int, int ) ) );
416     filteredData_->runSearch( QRegExp( "0000.4" ) );
417 
418     for ( int i = 0; i < 1; i++ ) {
419         waitSearchProgressed();
420         signalSearchProgressedRead();
421     }
422 
423     // We should have the result of the search and the marks
424     waitSearchProgressed();
425     QCOMPARE( filteredData_->getNbLine(), 12LL );
426     signalSearchProgressedRead();
427 
428     QString startline = "LOGDATA is a part of glogg, we are going to test it thoroughly, this is line ";
429 
430     QCOMPARE( filteredData_->getLineString(0), startline + "000004" );
431     QCOMPARE( filteredData_->getLineString(1), startline + "000010" );
432     QCOMPARE( filteredData_->getLineString(2), startline + "000014" );
433     QCOMPARE( filteredData_->getLineString(3), startline + "000024" );
434     QCOMPARE( filteredData_->getLineString(4), startline + "000025" );
435     QCOMPARE( filteredData_->getLineString(5), startline + "000034" );
436     QCOMPARE( filteredData_->getLineString(6), startline + "000044" );
437     QCOMPARE( filteredData_->getLineString(7), startline + "000054" );
438     QCOMPARE( filteredData_->getLineString(8), startline + "000064" );
439     QCOMPARE( filteredData_->getLineString(9), startline + "000074" );
440     QCOMPARE( filteredData_->getLineString(10), startline + "000084" );
441     QCOMPARE( filteredData_->getLineString(11), startline + "000094" );
442 
443     filteredData_->setVisibility( LogFilteredData::MatchesOnly );
444 
445     QCOMPARE( filteredData_->getNbLine(), 10LL );
446     QCOMPARE( filteredData_->getLineString(0), startline + "000004" );
447     QCOMPARE( filteredData_->getLineString(1), startline + "000014" );
448     QCOMPARE( filteredData_->getLineString(2), startline + "000024" );
449     QCOMPARE( filteredData_->getLineString(3), startline + "000034" );
450     QCOMPARE( filteredData_->getLineString(4), startline + "000044" );
451     QCOMPARE( filteredData_->getLineString(5), startline + "000054" );
452     QCOMPARE( filteredData_->getLineString(6), startline + "000064" );
453     QCOMPARE( filteredData_->getLineString(7), startline + "000074" );
454     QCOMPARE( filteredData_->getLineString(8), startline + "000084" );
455     QCOMPARE( filteredData_->getLineString(9), startline + "000094" );
456 
457     filteredData_->setVisibility( LogFilteredData::MarksOnly );
458 
459     QCOMPARE( filteredData_->getNbLine(), 3LL );
460     QCOMPARE( filteredData_->getLineString(0), startline + "000010" );
461     QCOMPARE( filteredData_->getLineString(1), startline + "000025" );
462     QCOMPARE( filteredData_->getLineString(2), startline + "000044" );
463 
464     // Another test with marks only
465     filteredData_->clearSearch();
466     filteredData_->clearMarks();
467     filteredData_->setVisibility( LogFilteredData::MarksOnly );
468 
469     filteredData_->addMark(18);
470     filteredData_->addMark(19);
471     filteredData_->addMark(20);
472 
473     QCOMPARE( filteredData_->getMatchingLineNumber(0), 18LL );
474     QCOMPARE( filteredData_->getMatchingLineNumber(1), 19LL );
475     QCOMPARE( filteredData_->getMatchingLineNumber(2), 20LL );
476 
477     QApplication::quit();
478 }
479 
480 void TestLogFilteredData::lineLength()
481 {
482     logData_ = new LogData();
483 
484     // Register for notification file is loaded
485     connect( logData_, SIGNAL( loadingFinished( bool ) ),
486             this, SLOT( loadingFinished() ) );
487 
488     filteredData_ = logData_->getNewFilteredData();
489     connect( filteredData_, SIGNAL( searchProgressed( int, int ) ),
490             this, SLOT( searchProgressed( int, int ) ) );
491 
492     QFuture<void> future = QtConcurrent::run(this, &TestLogFilteredData::lineLengthTest);
493 
494     QApplication::exec();
495 
496     disconnect( filteredData_, 0 );
497     disconnect( logData_, 0 );
498 
499     delete filteredData_;
500     delete logData_;
501 }
502 
503 void TestLogFilteredData::lineLengthTest()
504 {
505     // Line length tests
506 
507     logData_->attachFile( TMPDIR "/length_test.txt" );
508     // Wait for the loading to be done
509     waitLoadingFinished();
510     QCOMPARE( logData_->getNbLine(), 4LL );
511     signalLoadingFinishedRead();
512 
513     // Performs a search (the two middle lines matche)
514     filteredData_->setVisibility( LogFilteredData::MatchesOnly );
515     filteredData_->runSearch( QRegExp( "longer" ) );
516 
517     std::pair<int,int> progress;
518     do {
519         progress = waitSearchProgressed();
520         signalSearchProgressedRead();
521         QWARN("progress");
522     } while ( progress.second < 100 );
523 
524     filteredData_->addMark( 3 );
525 
526     QCOMPARE( filteredData_->getNbLine(), 2LL );
527     QCOMPARE( filteredData_->getMaxLength(), 40 );
528 
529     filteredData_->setVisibility( LogFilteredData::MarksAndMatches );
530     QCOMPARE( filteredData_->getNbLine(), 3LL );
531     QCOMPARE( filteredData_->getMaxLength(), 103 );
532 
533     filteredData_->setVisibility( LogFilteredData::MarksOnly );
534     QCOMPARE( filteredData_->getNbLine(), 1LL );
535     QCOMPARE( filteredData_->getMaxLength(), 103 );
536 
537     filteredData_->addMark( 0 );
538     QCOMPARE( filteredData_->getNbLine(), 2LL );
539     QCOMPARE( filteredData_->getMaxLength(), 103 );
540     filteredData_->deleteMark( 3 );
541     QCOMPARE( filteredData_->getNbLine(), 1LL );
542     QCOMPARE( filteredData_->getMaxLength(), 27 );
543 
544     filteredData_->setVisibility( LogFilteredData::MarksAndMatches );
545     QCOMPARE( filteredData_->getMaxLength(), 40 );
546 
547     QApplication::quit();
548 }
549 
550 //
551 // Private functions
552 //
553 void TestLogFilteredData::loadingFinished()
554 {
555     QMutexLocker locker( &loadingFinishedMutex_ );
556 
557     QWARN("loadingFinished");
558     loadingFinished_received_ = true;
559     loadingFinished_read_ = false;
560 
561     loadingFinishedCondition_.wakeOne();
562 
563     // Wait for the test thread to read the signal
564     while ( ! loadingFinished_read_ )
565         loadingFinishedCondition_.wait( locker.mutex() );
566 }
567 
568 void TestLogFilteredData::searchProgressed( int nbMatches, int completion )
569 {
570     QMutexLocker locker( &searchProgressedMutex_ );
571 
572     QWARN("searchProgressed");
573     searchProgressed_received_ = true;
574     searchProgressed_read_ = false;
575 
576     searchLastMatches_ = nbMatches;
577     searchLastProgress_ = completion;
578 
579     searchProgressedCondition_.wakeOne();
580 
581     // Wait for the test thread to read the signal
582     while ( ! searchProgressed_read_ )
583         searchProgressedCondition_.wait( locker.mutex() );
584 }
585 
586 std::pair<int,int> TestLogFilteredData::waitSearchProgressed()
587 {
588     QMutexLocker locker( &searchProgressedMutex_ );
589 
590     while ( ! searchProgressed_received_ )
591         searchProgressedCondition_.wait( locker.mutex() );
592 
593     QWARN("searchProgressed Received");
594 
595     return std::pair<int,int>(searchLastMatches_, searchLastProgress_);
596 }
597 
598 void TestLogFilteredData::waitLoadingFinished()
599 {
600     QMutexLocker locker( &loadingFinishedMutex_ );
601 
602     while ( ! loadingFinished_received_ )
603         loadingFinishedCondition_.wait( locker.mutex() );
604 
605     QWARN("loadingFinished Received");
606 }
607 
608 void TestLogFilteredData::signalSearchProgressedRead()
609 {
610     QMutexLocker locker( &searchProgressedMutex_ );
611 
612     searchProgressed_received_ = false;
613     searchProgressed_read_ = true;
614     searchProgressedCondition_.wakeOne();
615 }
616 
617 void TestLogFilteredData::signalLoadingFinishedRead()
618 {
619     QMutexLocker locker( &loadingFinishedMutex_ );
620 
621     loadingFinished_received_ = false;
622     loadingFinished_read_ = true;
623     loadingFinishedCondition_.wakeOne();
624 }
625 
626 bool TestLogFilteredData::generateDataFiles()
627 {
628     char newLine[90];
629 
630     QFile file( TMPDIR "/mediumlog.txt" );
631     if ( file.open( QIODevice::WriteOnly ) ) {
632         for (int i = 0; i < ML_NB_LINES; i++) {
633             snprintf(newLine, 89, ml_format, i);
634             file.write( newLine, qstrlen(newLine) );
635         }
636     }
637     else {
638         return false;
639     }
640     file.close();
641 
642     file.setFileName( TMPDIR "/smalllog.txt" );
643     if ( file.open( QIODevice::WriteOnly ) ) {
644         for (int i = 0; i < SL_NB_LINES; i++) {
645             snprintf(newLine, 89, sl_format, i);
646             file.write( newLine, qstrlen(newLine) );
647         }
648     }
649     else {
650         return false;
651     }
652     file.close();
653 
654     file.setFileName( TMPDIR "/length_test.txt" );
655     if ( file.open( QIODevice::WriteOnly ) ) {
656         file.write( "This line is 27 characters.\n" );
657         file.write( "This line is longer: 36 characters.\n" );
658         file.write( "This line is even longer: 40 characters.\n" );
659         file.write( "This line is very long, it's actually hard to count but it is\
660  probably something around 103 characters.\n" );
661     }
662     file.close();
663 
664     return true;
665 }
666