Repository navigation
Expand file tree
/
Copy pathLiveCaptureProcessor.cpp
More file actions
2158 lines (1954 loc) · 86.2 KB
/
Copy pathLiveCaptureProcessor.cpp
File metadata and controls
2158 lines (1954 loc) · 86.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include <windows.h>
#include <cstring>
#include <cassert>
#include <fstream>
#include <string>
#include "glhook_injector.h"
#include "livecaptureprocessor.h"
#include <array>
#include <algorithm>
#include <tuple>
#include <cstdarg>
#include <cstdio>
#include "meshdata.h"
#include "worlddata.h"
#include "debugout.h"
#include <GeographicLib/LocalCartesian.hpp>
#include <GLFW/glfw3.h>
using namespace std;
// ---------------------------------------------------------------------------
// Capture diagnostics: C:\Users\Public\meshtool_capture.log
// ---------------------------------------------------------------------------
static void CapLog(const char* fmt, ...)
{
FILE* f = fopen("C:\\Users\\Public\\meshtool_capture.log", "a");
if (!f) return;
va_list args;
va_start(args, fmt);
vfprintf(f, fmt, args);
va_end(args);
fclose(f);
}
// Google Earth's world unit is one Earth radius.
static const double kMetresPerUnit = 6378137.0;
static int g_logged_draws = 0; // draw calls logged in the current frame
static const int kMaxLoggedDraws = 40;
static int g_meshes_from_frame = 0;
// Size in bytes of one component of a GL vertex attribute / index type; 0 if unsupported.
static uint32_t GLTypeSize(uint32_t type)
{
switch (type)
{
case 0x1400: case 0x1401: return 1; // BYTE, UNSIGNED_BYTE
case 0x1402: case 0x1403: return 2; // SHORT, UNSIGNED_SHORT
case 0x1404: case 0x1405: case 0x1406: return 4; // INT, UNSIGNED_INT, FLOAT
case 0x140A: return 8; // DOUBLE
}
return 0;
}
// One component converted to float the way GL feeds it to the shader.
static float ReadGLComponent(const char* p, uint32_t type, bool normalized)
{
switch (type)
{
case 0x1400: { int8_t v = *(const int8_t*)p; return normalized ? (v / 127.0f < -1.0f ? -1.0f : v / 127.0f) : float(v); }
case 0x1401: { uint8_t v = *(const uint8_t*)p; return normalized ? v / 255.0f : float(v); }
case 0x1402: { int16_t v = *(const int16_t*)p; return normalized ? (v / 32767.0f < -1.0f ? -1.0f : v / 32767.0f) : float(v); }
case 0x1403: { uint16_t v = *(const uint16_t*)p; return normalized ? v / 65535.0f : float(v); }
case 0x1404: { int32_t v = *(const int32_t*)p; return normalized ? float(v / 2147483647.0) : float(v); }
case 0x1405: { uint32_t v = *(const uint32_t*)p; return normalized ? float(v / 4294967295.0) : float(v); }
case 0x1406: return *(const float*)p;
case 0x140A: return float(*(const double*)p);
}
return 0.0f;
}
// Reads every vertex of an attribute stream from its buffer into `out`
// (up to 4 components each, missing ones = 0). Stride 0 means tightly packed.
static size_t ReadAttributeStream(const std::vector<char>& buf, const VertexAttrib& a, std::vector<std::array<float, 4>>& out)
{
uint32_t comp_size = GLTypeSize(uint32_t(a.data_type));
uint32_t comps = a.num_elements < 1 ? 1 : (a.num_elements > 4 ? 4 : a.num_elements);
if (comp_size == 0)
return 0;
uint32_t elem_size = comp_size * comps;
uint32_t stride = a.stride ? a.stride : elem_size;
const char* p = buf.data() + a.start_offset;
const char* end = buf.data() + buf.size();
if (a.start_offset >= buf.size())
return 0;
for (; p + elem_size <= end; p += stride)
{
std::array<float, 4> v = { 0.0f, 0.0f, 0.0f, 0.0f };
for (uint32_t c = 0; c < comps; c++)
v[c] = ReadGLComponent(p + c * comp_size, uint32_t(a.data_type), a.is_normalized != 0);
out.push_back(v);
}
return out.size();
}
// ============================================================================
// Helper: convert matrix4f to matrix4d (same as in GpaDumpAnalyzeTool)
// ============================================================================
static void Matrix4fToMatrix4d(const core::matrix4f& src_mat, core::matrix4d& dst_mat)
{
for (int r = 0; r < 4; r++)
{
core::vec4f row = src_mat.get_row(r);
dst_mat.set_row(r, core::vec4d(double(row.x), double(row.y), double(row.z), double(row.w)));
}
}
// ============================================================================
// LiveCaptureProcessor
// ============================================================================
LiveCaptureProcessor::LiveCaptureProcessor()
: m_mapping(nullptr)
, m_shared_mem(nullptr)
, m_header(nullptr)
, m_ring_base(nullptr)
, m_event_ready(nullptr)
, m_current_bind_texture(-1)
, m_current_bind_buffer(-1)
, m_current_texture_slot(0)
, m_current_group(nullptr)
, m_output_batch(nullptr)
, m_has_first_matrix(false)
{
memset(m_bind_buffer_list, -1, sizeof(m_bind_buffer_list));
}
LiveCaptureProcessor::~LiveCaptureProcessor()
{
if (m_shared_mem) UnmapViewOfFile(m_shared_mem);
if (m_mapping) CloseHandle(m_mapping);
if (m_event_ready) CloseHandle(m_event_ready);
// Clean up texture store (except textures now owned by captured groups)
for (auto& pair : m_texture_store)
if (!m_textures_handed_out.count(pair.second))
delete pair.second;
}
void LiveCaptureProcessor::processFrame(bool keepMeshes)
{
// Open shared memory if not already open
if (!m_shared_mem)
{
m_mapping = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, GLCAPTURE_SHARED_MEM_NAME);
if (!m_mapping)
{
if (m_on_capture_error) m_on_capture_error("Cannot open shared memory - is Google Earth running with hook?");
return;
}
m_shared_mem = MapViewOfFile(m_mapping, FILE_MAP_ALL_ACCESS, 0, 0, GLCAPTURE_SHARED_MEM_SIZE);
if (!m_shared_mem)
{
CloseHandle(m_mapping);
m_mapping = nullptr;
if (m_on_capture_error) m_on_capture_error("Cannot map shared memory");
return;
}
m_header = reinterpret_cast<GLCaptureHeader*>(m_shared_mem);
m_ring_base = reinterpret_cast<char*>(m_shared_mem) + GLCAPTURE_HEADER_SIZE;
m_event_ready = OpenEventA(EVENT_ALL_ACCESS, FALSE, GLCAPTURE_EVENT_READY);
if (m_on_connection_changed) m_on_connection_changed(true);
}
// Check if hook is connected
if (!(m_header->status_flags & GLCAPTURE_STATUS_CONNECTED))
{
if (m_on_capture_error) m_on_capture_error("Hook DLL not connected");
return;
}
m_last = FrameResult();
uint64_t read_pos = m_header->read_offset;
const uint64_t write_pos = m_header->write_offset;
// Pass 1: find the end of the newest complete frame. More than one may be
// queued (two captures can finish before we wake up: the ready event only
// remembers one signal), and the newest is the one GE shows right now -
// older ones would make every capture lag one behind. Records after the
// last frame end belong to a capture still being written; leave them.
uint64_t last_end = 0;
int complete_frames = 0;
for (uint64_t p = read_pos; p != write_pos;)
{
if (p >= GLCAPTURE_RING_SIZE) { p = 0; continue; }
const GLCaptureRecord* rec = reinterpret_cast<const GLCaptureRecord*>(m_ring_base + p);
if (rec->cmd_id == CMD_NONE) { p = 0; continue; }
if (rec->total_size == 0) break; // corrupt; don't spin
p += rec->total_size;
if (p >= GLCAPTURE_RING_SIZE) p = 0;
if (rec->cmd_id == CMD_FRAME_END)
{
last_end = p;
complete_frames++;
}
}
CapLog("=== processFrame: read=%llu write=%llu status=0x%X, %d complete frame(s) queued\n",
(unsigned long long)read_pos, (unsigned long long)write_pos, m_header->status_flags, complete_frames);
const uint32_t status = m_header->status_flags;
if (complete_frames == 0)
{
// A capture that overflowed without its frame end (hooks before the
// frame-end reserve) blocks the ring for good: nothing complete to
// read, no room to finish. Drop it; each capture resends what it needs.
if ((status & GLCAPTURE_STATUS_OVERFLOW) && !(status & GLCAPTURE_STATUS_CAPTURING))
{
CapLog(" ring full with an unfinished capture: dropped it\n");
m_header->read_offset = write_pos;
m_header->status_flags &= ~GLCAPTURE_STATUS_OVERFLOW;
m_last.dropped = true;
}
return;
}
if (status & GLCAPTURE_STATUS_OVERFLOW)
{
CapLog(" capture overflowed the ring: partly recorded\n");
m_last.truncated = true;
}
auto start_frame = [this]() {
m_current_group = new GroupMeshData;
g_logged_draws = 0;
g_meshes_from_frame = 0;
m_has_first_matrix = false;
m_matrix_stack.clear();
m_up_accum = core::vec3d(0.0, 0.0, 0.0);
// m_tile_size / m_tile_key persist: the merged capture's meshes keep theirs.
};
start_frame();
std::map<uint32_t, uint32_t> record_counts;
// Pass 2: replay every queued record (GL state carries across frames) but
// keep only the newest frame's meshes.
while (read_pos != last_end)
{
if (read_pos >= GLCAPTURE_RING_SIZE) { read_pos = 0; continue; }
const GLCaptureRecord* record = reinterpret_cast<const GLCaptureRecord*>(m_ring_base + read_pos);
if (record->cmd_id == CMD_NONE) { read_pos = 0; continue; }
if (record->total_size == 0) break;
if (record->cmd_id == CMD_FRAME_END)
{
read_pos += record->total_size;
if (read_pos >= GLCAPTURE_RING_SIZE) read_pos = 0;
if (read_pos != last_end)
{
// An older queued frame: drop its meshes and start over.
CapLog(" skipped an older queued frame (%d meshes)\n", int(m_current_group->meshes.size()));
for (MeshData* mesh : m_current_group->meshes)
{
m_tile_size.erase(mesh);
m_tile_origin.erase(mesh);
m_tile_key.erase(mesh);
delete mesh;
}
delete m_current_group;
start_frame();
record_counts.clear();
}
continue;
}
const char* payload = m_ring_base + read_pos + sizeof(GLCaptureRecord);
record_counts[record->cmd_id]++;
ProcessRecord(record, payload);
read_pos += record->total_size;
if (read_pos >= GLCAPTURE_RING_SIZE) read_pos = 0;
}
// Update read position
m_header->read_offset = read_pos;
CapLog(" records:");
for (auto& rc : record_counts)
CapLog(" 0x%04X x%u", rc.first, rc.second);
CapLog("\n buffers known=%u textures known=%u meshes built=%d%s\n",
unsigned(m_buffer_store.size()), unsigned(m_texture_store.size()), g_meshes_from_frame,
(m_header->status_flags & GLCAPTURE_STATUS_OVERFLOW) ? " (hook reported RING OVERFLOW)" : "");
// Add captured meshes to output
int mesh_count = (int)m_current_group->meshes.size();
if (keepMeshes && mesh_count > 0 && m_output_batch)
{
// Where this capture goes. Every capture is placed in one scene frame:
// East/North/Up metres at the batch's GPS reference when GE reported
// its view, otherwise a frame derived from the first capture's camera.
// A capture that overlaps or borders one of the captured areas is
// merged into it, through X (this frame's eye space -> the area's
// reference eye space): from tiles both contain (exact) or else from
// both GPS views. Any other capture starts a new area.
const bool geo = m_geo_view.valid;
CheckLookAt(m_current_group);
// This capture's footprint in the scene when placed through X into frame f.
auto footprint = [&](const core::matrix4d& Xm, const GroundFrame& f) {
core::bounds3d box;
for (MeshData* mesh : m_current_group->meshes)
{
if (!mesh->bbox_ws.b_valid) continue;
for (int c = 0; c < 8; c++)
{
core::vec3d p((c & 1) ? mesh->bbox_ws.bb_max.x : mesh->bbox_ws.bb_min.x,
(c & 2) ? mesh->bbox_ws.bb_max.y : mesh->bbox_ws.bb_min.y,
(c & 4) ? mesh->bbox_ws.bb_max.z : mesh->bbox_ws.bb_min.z);
box += ToGround(p, &Xm, f);
}
}
return box;
};
// GPS placement only (shared tiles prove the overlap): does the capture
// border the area, at a comparable level of detail?
auto neighbours = [&](const core::bounds3d& probe_box, const Area& a) {
const core::bounds3d& old_box = a.group->bbox_ws;
const double kNeighbourGap = 100.0; // metres between footprints that still count as neighbours
double gap_x = (std::max)(probe_box.bb_min.x - old_box.bb_max.x, old_box.bb_min.x - probe_box.bb_max.x);
double gap_y = (std::max)(probe_box.bb_min.y - old_box.bb_max.y, old_box.bb_min.y - probe_box.bb_max.y);
// A frame grabbed while GE was still zoomed out (planet-sized
// tiles) must not join a street-level area or vice versa. Compare
// the finest tiles, not the footprints: a tilted view reaches the
// horizon, so its footprint says little about the zoom.
const double kMaxLevelGap = 5.0; // levels of detail, each halves the tile edge
double new_tile = FinestTile(m_current_group->meshes), old_tile = FinestTile(a.group->meshes);
double levels = (new_tile > 0.0 && old_tile > 0.0) ? fabs(log2(new_tile / old_tile)) : 0.0;
bool ok = probe_box.b_valid && old_box.b_valid && gap_x <= kNeighbourGap && gap_y <= kNeighbourGap &&
levels <= kMaxLevelGap;
CapLog(" GPS placement vs area %d: footprint gap %.1f x %.1f m, finest tile %.1f vs %.1f m (%.1f levels) -> %s\n",
int(&a - m_areas.data()) + 1, (std::max)(gap_x, 0.0), (std::max)(gap_y, 0.0), new_tile, old_tile,
levels, ok ? "MERGE" : "no");
return ok;
};
auto updateBatchBounds = [&]() {
m_output_batch->bbox_ws.Reset();
m_output_batch->bbox_gps.Reset();
for (const GroupMeshData* g : m_output_batch->group_meshes)
{
if (g->bbox_ws.b_valid) m_output_batch->bbox_ws += g->bbox_ws;
if (g->bbox_gps.b_valid) m_output_batch->bbox_gps += g->bbox_gps;
}
};
// Which area the capture joins, and X: this frame's eye space -> that
// area's reference eye space.
Area* target = nullptr;
core::matrix4d X;
std::string placement; // how X was found (debug view)
// 1. Shared tiles: exact, and proof that the captures overlap, so the
// capture merges whatever its zoom or tilt. The best-supported area wins.
int best_support = 0;
for (Area& a : m_areas)
{
core::matrix4d Xa;
int support = 0;
if (RegisterToReference(m_current_group, a, Xa, support) && support > best_support)
{
best_support = support;
target = &a;
X = Xa;
}
}
// GE's view in the scene frame (the batch's GPS reference), for GPS placement.
GroundFrame geo_in_scene;
if (geo && m_output_batch->is_georeferenced)
geo_in_scene = GroundFrameFromGeo(m_output_batch->reference_pos.x, m_output_batch->reference_pos.y);
if (target)
{
placement = "shared tiles (" + std::to_string(best_support) + ")";
CapLog(" placement by shared tiles: %d agreeing -> MERGE into area %d\n",
best_support, int(target - m_areas.data()) + 1);
if (geo_in_scene.valid && target->frame.geo)
{
// Both placements are available: how well does GPS agree with the exact one?
core::matrix4d Xg = FrameMatrix(geo_in_scene) * inverse(FrameMatrix(target->frame));
double dx = (X(3, 0) - Xg(3, 0)), dy = (X(3, 1) - Xg(3, 1)), dz = (X(3, 2) - Xg(3, 2));
const double kDeg = 180.0 / 3.14159265358979323846;
CapLog(" GPS vs shared-tile placement of the camera: %.2f m apart, rotation %.2f vs %.2f deg\n",
sqrt(dx * dx + dy * dy + dz * dz) * kMetresPerUnit,
atan2(Xg(0, 1), Xg(0, 0)) * kDeg, atan2(X(0, 1), X(0, 0)) * kDeg);
}
}
// 2. GPS: the newest area the capture borders.
else if (geo_in_scene.valid)
{
for (size_t k = m_areas.size(); k-- > 0;)
{
Area& a = m_areas[k];
if (!a.frame.geo) continue;
core::matrix4d Xa = FrameMatrix(geo_in_scene) * inverse(FrameMatrix(a.frame));
if (neighbours(footprint(Xa, a.frame), a))
{
target = &a;
X = Xa;
placement = "GPS";
break;
}
}
}
if (!target && !m_areas.empty())
CapLog(" placement: no area shares tiles with it or borders it -> new area\n");
if (target)
{
GroupMeshData* group = target->group;
// Tiles the area already has are identical data: drop them.
int duplicates = 0;
for (size_t k = 0; k < m_current_group->meshes.size();)
{
MeshData* mesh = m_current_group->meshes[k];
auto key = m_tile_key.find(mesh);
if (key != m_tile_key.end() && target->key_mesh.count(key->second))
{
m_tile_key.erase(mesh);
m_tile_size.erase(mesh);
m_tile_origin.erase(mesh);
delete mesh; // never uploaded
m_current_group->meshes.erase(m_current_group->meshes.begin() + k);
duplicates++;
continue;
}
k++;
}
ApplyGroundFrame(m_current_group, &X, target->frame);
CaptureInfo capture = DescribeCapture(m_current_group->meshes, &X, target->frame,
int32_t(group->captures.size()), placement);
capture.duplicates = duplicates;
group->captures.push_back(capture);
HandOutTextures(m_current_group, uint32_t(group->loaded_textures.size()));
for (core::Texture2DInfo* tex : m_current_group->loaded_textures)
group->loaded_textures.push_back(tex);
for (MeshData* mesh : m_current_group->meshes)
{
auto key = m_tile_key.find(mesh);
if (key != m_tile_key.end())
{
target->key_mesh[key->second] = mesh;
target->ref_tile_mv[key->second] = mesh->dumpped_matrix * X;
}
group->meshes.push_back(mesh);
}
int added = int(m_current_group->meshes.size());
m_current_group->meshes.clear();
m_current_group->loaded_textures.clear();
delete m_current_group;
m_current_group = nullptr;
// Finer tiles from either capture now hide coarser ones from both.
RemoveCoveredLods(*target);
group->bbox_ws.Reset();
for (MeshData* mesh : group->meshes)
if (mesh->bbox_ws.b_valid) group->bbox_ws += mesh->bbox_ws;
SeparateNoGpsAreas();
updateBatchBounds();
CapLog(" merged into area %d: %d tiles added, %d duplicates dropped, %u meshes total\n",
int(target - m_areas.data()) + 1, added, duplicates, unsigned(group->meshes.size()));
m_last.new_data = true;
m_last.merged = true;
m_last.group = group;
if (m_on_frame_captured) m_on_frame_captured(int(group->meshes.size()));
return;
}
// New area. Areas are never replaced: GPS places them all in one scene
// frame, and one captured without GPS (no true location) is set down
// beside the rest. Their tiles stay known, so later captures can still
// merge into any area. Only an overview (GE zoomed far out, e.g. while
// flying in) is dropped by the next capture.
const double kOverviewTile = 1000.0; // metres: finest tile coarser than this = overview
for (size_t k = 0; k < m_areas.size();)
{
double finest = FinestTile(m_areas[k].group->meshes);
if (!(finest > kOverviewTile))
{
k++;
continue;
}
CapLog(" dropping overview area %zu (finest tile %.0f m)\n", k + 1, finest);
GroupMeshData* old = m_areas[k].group;
for (MeshData* mesh : old->meshes)
{
m_tile_size.erase(mesh);
m_tile_origin.erase(mesh);
m_tile_key.erase(mesh);
}
auto& groups = m_output_batch->group_meshes;
groups.erase(std::remove(groups.begin(), groups.end(), old), groups.end());
m_last.discarded.push_back(old); // the caller frees it
m_areas.erase(m_areas.begin() + k);
}
if (m_output_batch->group_meshes.empty())
m_output_batch->is_georeferenced = false; // a fresh scene takes this capture's GPS origin
GroundFrame frame;
if (geo)
{
if (!m_output_batch->is_georeferenced)
{
// The scene's GPS origin is this capture's look-at point. Areas
// captured without GPS keep their (arbitrary) place and are
// moved aside below if they overlap.
const double lon0 = m_geo_view.hasLookAt ? m_geo_view.laLon : m_geo_view.lookLon;
const double lat0 = m_geo_view.hasLookAt ? m_geo_view.laLat : m_geo_view.lookLat;
m_output_batch->reference_pos = core::vec2d(lon0, lat0);
m_output_batch->is_georeferenced = true;
}
frame = GroundFrameFromGeo(m_output_batch->reference_pos.x, m_output_batch->reference_pos.y);
}
else
{
CapLog(" no GPS view from Google Earth (KML view link not loaded?): using a camera-based frame\n");
frame = ComputeGroundFrame(m_current_group);
}
m_areas.push_back(Area());
Area& area = m_areas.back();
area.group = m_current_group;
area.frame = frame;
m_current_group->no_gps = !geo;
for (MeshData* mesh : m_current_group->meshes)
{
auto key = m_tile_key.find(mesh);
if (key != m_tile_key.end())
{
area.key_mesh[key->second] = mesh;
area.ref_tile_mv[key->second] = mesh->dumpped_matrix; // reference eye space = this frame's
}
}
ApplyGroundFrame(m_current_group, nullptr, frame);
m_current_group->captures.push_back(
DescribeCapture(m_current_group->meshes, nullptr, frame, 0, geo ? "first capture (GPS)" : "first capture (no GPS)"));
RemoveCoveredLods(area);
HandOutTextures(m_current_group, 0);
m_output_batch->group_meshes.push_back(m_current_group);
m_last.group = m_current_group;
m_current_group = nullptr;
SeparateNoGpsAreas();
updateBatchBounds();
CapLog(" new area %d of %zu%s\n", int(m_areas.size()), m_areas.size(), geo ? "" : " (no GPS)");
m_last.new_data = true;
if (m_on_frame_captured) m_on_frame_captured(int(m_last.group->meshes.size()));
}
else
{
for (MeshData* mesh : m_current_group->meshes)
{
m_tile_size.erase(mesh);
m_tile_origin.erase(mesh);
m_tile_key.erase(mesh);
delete mesh;
}
delete m_current_group;
m_current_group = nullptr;
if (m_on_frame_captured) m_on_frame_captured(0);
}
}
LiveCaptureProcessor::GroundFrame LiveCaptureProcessor::GroundFrameFromGeo(double lon0, double lat0) const
{
GroundFrame f;
const GeoView& v = m_geo_view;
if (!v.valid)
return f;
// GE's LookAt, in East/North/Up metres at the look-at point L (at altitude
// 0, which is what `range` is measured from): the camera sits `range`
// away, tilted `tilt` from vertical, looking along `heading`; GE cameras
// don't roll, so screen-right stays horizontal. A link written by an older
// MeshTool has no LookAt point; the terrain point is the same for tilt 0.
const double la_lon = v.hasLookAt ? v.laLon : v.lookLon;
const double la_lat = v.hasLookAt ? v.laLat : v.lookLat;
const double deg = 3.14159265358979323846 / 180.0;
const double h = v.heading * deg, t = v.tilt * deg;
const core::vec3d right(cos(h), -sin(h), 0.0);
const core::vec3d horiz(sin(h), cos(h), 0.0);
const core::vec3d cam = core::vec3d(-sin(t) * horiz.x, -sin(t) * horiz.y, cos(t)) * v.range;
const core::vec3d fwd(sin(t) * horiz.x, sin(t) * horiz.y, -cos(t));
const core::vec3d eye_up = cross(right, fwd);
const core::vec3d eye_z = fwd * -1.0; // eye space looks down -Z
// L's ENU -> scene ENU at (lat0, lon0, 0): L's position T and rotation M.
GeographicLib::LocalCartesian scene(lat0, lon0, 0.0);
double tx, ty, tz;
std::vector<double> M(9);
scene.Forward(la_lat, la_lon, 0.0, tx, ty, tz, M);
auto rotate = [&M](const core::vec3d& p) {
return core::vec3d(M[0] * p.x + M[1] * p.y + M[2] * p.z,
M[3] * p.x + M[4] * p.y + M[5] * p.z,
M[6] * p.x + M[7] * p.y + M[8] * p.z);
};
// ground = metres * Rot * p_eye + trans, Rot's columns = eye axes in scene ENU.
core::vec3d c0 = rotate(right), c1 = rotate(eye_up), c2 = rotate(eye_z);
core::vec3d trans = core::vec3d(tx, ty, tz) + rotate(cam);
f.x_axis = core::vec3d(c0.x, c1.x, c2.x); // rows of Rot, as eye-space vectors
f.y_axis = core::vec3d(c0.y, c1.y, c2.y);
f.up = core::vec3d(c0.z, c1.z, c2.z);
// ground = metres * Rot * (p - origin) => origin = -Rot^T * trans / metres,
// and Rot^T's rows are Rot's columns.
f.origin = core::vec3d(dot(c0, trans), dot(c1, trans), dot(c2, trans)) * (-1.0 / kMetresPerUnit);
f.valid = true;
f.geo = true;
// Cross-check against GE's own camera position (its altitude reference
// may differ, so horizontal and vertical are reported separately).
double cx, cy, cz;
scene.Forward(v.camLat, v.camLon, v.camAlt, cx, cy, cz);
CapLog(" GPS view: look-at %.7f, %.7f%s range %.1f m heading %.2f tilt %.2f terrain %.1f m (age %.1f s)\n"
" GPS camera check: computed vs GE camera differ %.2f m horizontally, %.2f m vertically\n",
la_lat, la_lon, v.hasLookAt ? "" : " (terrain point: old view link)", v.range, v.heading, v.tilt,
v.lookAlt, glfwGetTime() - v.time,
sqrt((cx - trans.x) * (cx - trans.x) + (cy - trans.y) * (cy - trans.y)), cz - trans.z);
return f;
}
core::matrix4d LiveCaptureProcessor::FrameMatrix(const GroundFrame& f)
{
// [g, 1] = [p, 1] * G with g = Rot * (p - origin) (row vectors).
core::matrix4d G;
const core::vec3d ax[3] = { f.x_axis, f.y_axis, f.up };
for (int k = 0; k < 3; k++)
{
for (int i = 0; i < 3; i++)
G(k, i) = ax[i][k];
G(k, 3) = 0.0;
}
for (int i = 0; i < 3; i++)
G(3, i) = -dot(f.origin, ax[i]);
G(3, 3) = 1.0;
return G;
}
void LiveCaptureProcessor::CheckLookAt(const GroupMeshData* group) const
{
// GE's look-at point is the terrain at screen centre, `range` metres down
// the camera's -Z axis. Find the captured vertex closest to that ray.
if (!m_geo_view.valid || !group)
return;
double best_angle = 1e9, best_depth = 0.0;
for (const MeshData* mesh : group->meshes)
{
if (!mesh->vertex_list) continue;
for (int i = 0; i < mesh->num_vertex; i++)
{
const core::vec3f& v = mesh->vertex_list[i];
core::vec3d p = core::vec3d(v.x, v.y, v.z) + mesh->translation;
if (p.z >= 0.0) continue;
double angle = sqrt(p.x * p.x + p.y * p.y) / -p.z;
if (angle < best_angle) { best_angle = angle; best_depth = -p.z * kMetresPerUnit; }
}
}
if (best_angle >= 0.01)
return;
// Where GE says the screen-centre terrain is, seen from the camera that
// GE's LookAt describes.
const GeoView& v = m_geo_view;
const double la_lon = v.hasLookAt ? v.laLon : v.lookLon;
const double la_lat = v.hasLookAt ? v.laLat : v.lookLat;
const double deg = 3.14159265358979323846 / 180.0;
const double h = v.heading * deg, t = v.tilt * deg;
GeographicLib::LocalCartesian at(la_lat, la_lon, 0.0);
double ex, ey, ez;
at.Forward(v.lookLat, v.lookLon, v.lookAlt, ex, ey, ez);
double cx = -sin(t) * sin(h) * v.range, cy = -sin(t) * cos(h) * v.range, cz = cos(t) * v.range;
double expected = sqrt((ex - cx) * (ex - cx) + (ey - cy) * (ey - cy) + (ez - cz) * (ez - cz));
CapLog(" GPS look-at check: captured surface at screen centre %.2f m away, GE view says %.2f m (diff %.2f m)\n",
best_depth, expected, best_depth - expected);
}
void LiveCaptureProcessor::ResetMerge()
{
m_areas.clear();
m_tile_size.clear();
m_tile_origin.clear();
m_tile_key.clear();
}
void LiveCaptureProcessor::HandOutTextures(GroupMeshData* group, uint32_t index_offset)
{
// Hand the group the textures its meshes sample. Meshes recorded the GL
// texture id; remap it to an index into the (eventual) loaded_textures,
// which is what the renderer and exporters expect.
std::map<uint32_t, uint32_t> gl_to_index;
int textured = 0;
for (MeshData* mesh : group->meshes)
{
uint32_t gl_id = mesh->idx_in_texture_list;
mesh->idx_in_texture_list = INVALID_VALUE;
auto st = m_texture_store.find(gl_id);
if (gl_id == 0 || st == m_texture_store.end() || !st->second->m_levelCount ||
!st->second->m_mips[0].m_imageData)
continue;
auto it = gl_to_index.find(gl_id);
if (it == gl_to_index.end())
{
it = gl_to_index.emplace(gl_id, index_offset + uint32_t(group->loaded_textures.size())).first;
group->loaded_textures.push_back(st->second);
m_textures_handed_out.insert(st->second);
}
mesh->idx_in_texture_list = it->second;
textured++;
}
// The group owns those now; a later capture re-sends them and must get
// fresh objects rather than overwrite this group's.
for (auto& g : gl_to_index)
m_texture_store.erase(g.first);
CapLog(" textures: %u used, %d of %u meshes textured\n",
unsigned(gl_to_index.size()), textured, unsigned(group->meshes.size()));
}
bool LiveCaptureProcessor::RegisterToReference(const GroupMeshData* group, const Area& area, core::matrix4d& X,
int& support) const
{
// Each tile both captures contain gives the camera-to-camera transform:
// p_eye_new * inv(MV_new) = p_tile, p_tile * MV_ref = p_eye_ref.
std::vector<core::matrix4d> candidates;
std::vector<std::pair<core::matrix4d, core::matrix4d>> pairs; // (MV_new, MV_ref) per candidate
for (const MeshData* mesh : group->meshes)
{
auto key = m_tile_key.find(const_cast<MeshData*>(mesh));
if (key == m_tile_key.end()) continue;
auto ref = area.ref_tile_mv.find(key->second);
if (ref == area.ref_tile_mv.end()) continue;
candidates.push_back(inverse(mesh->dumpped_matrix) * ref->second);
pairs.emplace_back(mesh->dumpped_matrix, ref->second);
}
// Consensus on where the new camera sits in the reference eye space
// (translation row); a hash collision or reused tile can't win the vote.
const double kTolerance = 3e-7; // Earth radii, about 2 m
int best = -1, best_votes = 0;
for (size_t i = 0; i < candidates.size(); i++)
{
int votes = 0;
for (size_t j = 0; j < candidates.size(); j++)
{
double dx = candidates[i](3, 0) - candidates[j](3, 0);
double dy = candidates[i](3, 1) - candidates[j](3, 1);
double dz = candidates[i](3, 2) - candidates[j](3, 2);
if (dx * dx + dy * dy + dz * dz < kTolerance * kTolerance)
votes++;
}
if (votes > best_votes) { best_votes = votes; best = int(i); }
}
support = best_votes;
if (best < 0 || best_votes < 3)
return false;
X = candidates[size_t(best)];
// Check: every shared tile's corners (tile space is the unit cube),
// placed through X, against where the reference capture has them. The
// vote only compared camera positions; this also catches a wrong rotation.
std::vector<double> residual; // metres, worst corner per agreeing tile
const core::vec3d bx = core::vec3d(candidates[size_t(best)](3, 0), candidates[size_t(best)](3, 1),
candidates[size_t(best)](3, 2));
for (size_t i = 0; i < candidates.size(); i++)
{
core::vec3d d = core::vec3d(candidates[i](3, 0), candidates[i](3, 1), candidates[i](3, 2)) - bx;
if (dot(d, d) >= kTolerance * kTolerance)
continue;
double worst = 0.0;
for (int c = 0; c < 8; c++)
{
core::vec4d corner((c & 1) ? 1.0 : 0.0, (c & 2) ? 1.0 : 0.0, (c & 4) ? 1.0 : 0.0, 1.0);
core::vec4d a = corner * pairs[i].first * X;
core::vec4d b = corner * pairs[i].second;
core::vec3d e(a.x - b.x, a.y - b.y, a.z - b.z);
worst = (std::max)(worst, length(e) * kMetresPerUnit);
}
residual.push_back(worst);
}
std::sort(residual.begin(), residual.end());
// Rotation of X about the eye's view axis (in-plane rotation between the two cameras).
const double yaw = atan2(X(0, 1), X(0, 0)) * 180.0 / 3.14159265358979323846;
CapLog(" shared-tile check: %zu tiles, corner error median %.3f m, 90%% %.3f m, max %.3f m; "
"camera rotation about view axis %.2f deg\n",
residual.size(), residual.empty() ? 0.0 : residual[residual.size() / 2],
residual.empty() ? 0.0 : residual[residual.size() * 9 / 10],
residual.empty() ? 0.0 : residual.back(), yaw);
return true;
}
LiveCaptureProcessor::GroundFrame LiveCaptureProcessor::ComputeGroundFrame(const GroupMeshData* group) const
{
GroundFrame f;
if (!group || !group->bbox_ws.b_valid)
return f;
// Eye space: GE camera at the origin, looking down -Z, screen-up = +Y.
f.origin = group->bbox_ws.GetCentroid();
f.up = core::vec3d(0.0, 1.0, 0.0);
double up_len = length(m_up_accum);
if (up_len > 0.0)
{
f.up = m_up_accum / up_len;
if (dot(f.up, f.origin) > 0.0) // the camera (eye-space origin) is above the ground
f.up = f.up * -1.0;
}
// Y axis: screen-up laid onto the ground plane, i.e. "away from the camera"
// when GE is tilted and screen-up when it looks straight down.
f.y_axis = core::vec3d(0.0, 1.0, 0.0) - f.up * f.up.y;
if (length(f.y_axis) < 1e-6)
f.y_axis = core::vec3d(0.0, 0.0, -1.0) - f.up * -f.up.z;
f.y_axis = normalize(f.y_axis);
f.x_axis = cross(f.y_axis, f.up);
f.valid = true;
CapLog(" ground frame: up=(%.4f %.4f %.4f) origin=(%.6g %.6g %.6g) units\n",
f.up.x, f.up.y, f.up.z, f.origin.x, f.origin.y, f.origin.z);
return f;
}
core::vec3d LiveCaptureProcessor::ToGround(const core::vec3d& p_eye, const core::matrix4d* X, const GroundFrame& f)
{
core::vec3d p = p_eye;
if (X)
{
core::vec4d q = core::vec4d(p.x, p.y, p.z, 1.0) * (*X);
p = core::vec3d(q.x, q.y, q.z);
}
core::vec3d d = p - f.origin;
return core::vec3d(dot(d, f.x_axis), dot(d, f.y_axis), dot(d, f.up)) * kMetresPerUnit;
}
CaptureInfo LiveCaptureProcessor::DescribeCapture(const std::vector<MeshData*>& meshes, const core::matrix4d* X,
const GroundFrame& f, int32_t capture_id,
const std::string& placement) const
{
CaptureInfo info;
info.placement = placement;
info.tiles_added = int(meshes.size());
for (MeshData* mesh : meshes)
{
mesh->capture_id = capture_id;
if (mesh->bbox_ws.b_valid)
info.footprint += mesh->bbox_ws;
}
// GE's camera is the eye-space origin, looking down -Z.
info.eye = ToGround(core::vec3d(0.0, 0.0, 0.0), X, f);
core::vec3d dir = ToGround(core::vec3d(0.0, 0.0, -1e-5), X, f) - info.eye; // ~64 m ahead
double len = length(dir);
dir = len > 0.0 ? dir / len : core::vec3d(0.0, 0.0, -1.0);
// Look target: where that ray meets the capture's mid height, or (looking
// at the horizon) a point as far out as the capture reaches.
const double ground_z = info.footprint.b_valid ? info.footprint.GetCentroid().z : 0.0;
double reach = info.footprint.b_valid ? length(info.footprint.GetDiagonal()) * 0.5 : 100.0;
if (dir.z < -0.02)
reach = (std::min)((info.eye.z - ground_z) / -dir.z, reach * 4.0);
info.target = info.eye + dir * reach;
CapLog(" capture #%d (%s): camera %.1f, %.1f, %.1f looking at %.1f, %.1f, %.1f %d tiles\n",
capture_id + 1, placement.c_str(), info.eye.x, info.eye.y, info.eye.z,
info.target.x, info.target.y, info.target.z, info.tiles_added);
return info;
}
void LiveCaptureProcessor::ApplyGroundFrame(GroupMeshData* group, const core::matrix4d* X, const GroundFrame& f)
{
if (!group || !f.valid)
return;
auto to_ground = [&](const core::vec3d& p) { return ToGround(p, X, f); };
group->bbox_ws.Reset();
for (MeshData* mesh : group->meshes)
{
if (!mesh || !mesh->vertex_list || mesh->num_vertex <= 0)
continue;
if (m_tile_size.count(mesh))
{
core::vec4d o = mesh->dumpped_matrix.get_row(3); // tile-space origin in eye space
m_tile_origin[mesh] = to_ground(core::vec3d(o.x, o.y, o.z));
}
uint32_t n = uint32_t(mesh->num_vertex);
std::vector<core::vec3d> pos(n);
mesh->bbox_ws.Reset();
for (uint32_t i = 0; i < n; i++)
{
const core::vec3f& v = mesh->vertex_list[i];
pos[i] = to_ground(core::vec3d(v.x, v.y, v.z) + mesh->translation);
mesh->bbox_ws += pos[i];
}
mesh->translation = mesh->bbox_ws.GetCentroid();
for (uint32_t i = 0; i < n; i++)
mesh->vertex_list[i] = core::vec3f(pos[i] - mesh->translation);
group->bbox_ws += mesh->bbox_ws;
}
if (group->bbox_ws.b_valid)
{
core::vec3d d = group->bbox_ws.GetDiagonal();
CapLog(" ground frame extent: %.1f x %.1f x %.1f m\n", d.x, d.y, d.z);
}
}
void LiveCaptureProcessor::ShiftArea(Area& area, const core::vec3d& d)
{
GroupMeshData* g = area.group;
auto shift = [&d](core::bounds3d& b) {
if (b.b_valid) { b.bb_min = b.bb_min + d; b.bb_max = b.bb_max + d; }
};
for (MeshData* mesh : g->meshes)
{
mesh->translation = mesh->translation + d; // vertices are relative to it: no re-upload
shift(mesh->bbox_ws);
auto o = m_tile_origin.find(mesh);
if (o != m_tile_origin.end()) o->second = o->second + d;
}
shift(g->bbox_ws);
for (SceneObject& obj : g->objects)
shift(obj.bbox_ws);
for (CaptureInfo& c : g->captures)
{
c.eye = c.eye + d;
c.target = c.target + d;
shift(c.footprint);
}
// Later captures placed into this area land shifted too.
area.frame.origin = area.frame.origin -
(area.frame.x_axis * d.x + area.frame.y_axis * d.y + area.frame.up * d.z) * (1.0 / kMetresPerUnit);
}
void LiveCaptureProcessor::SeparateNoGpsAreas()
{
// GPS areas stay where GPS put them; each no-GPS area (no true location)
// that overlaps what is placed so far moves to its +X side.
const double kGap = 50.0; // metres between a moved area and the rest
core::bounds3d placed;
for (const Area& a : m_areas)
if (!a.group->no_gps && a.group->bbox_ws.b_valid)
placed += a.group->bbox_ws;
for (Area& a : m_areas)
{
const core::bounds3d& b = a.group->bbox_ws;
if (!a.group->no_gps || !b.b_valid)
continue;
bool overlaps = placed.b_valid && b.bb_min.x < placed.bb_max.x + kGap && b.bb_max.x > placed.bb_min.x - kGap &&
b.bb_min.y < placed.bb_max.y + kGap && b.bb_max.y > placed.bb_min.y - kGap;
if (overlaps)
{
core::vec3d d(placed.bb_max.x + kGap - b.bb_min.x,
(placed.bb_min.y + placed.bb_max.y - b.bb_min.y - b.bb_max.y) * 0.5, 0.0);
CapLog(" no-GPS area %d moved %.0f m east, %.0f m north to sit beside the rest\n",
int(&a - m_areas.data()) + 1, d.x, d.y);
ShiftArea(a, d);
}
placed += a.group->bbox_ws;
}
}
double LiveCaptureProcessor::FinestTile(const std::vector<MeshData*>& meshes) const
{
double finest = 0.0;
for (MeshData* mesh : meshes)
{
auto it = m_tile_size.find(mesh);
if (it != m_tile_size.end() && it->second > 0.0 && (finest == 0.0 || it->second < finest))
finest = it->second;
}
return finest;
}
void LiveCaptureProcessor::RemoveCoveredLods(Area& area_info)
{
GroupMeshData* group = area_info.group;
if (!group)
return;
struct Item { MeshData* mesh; double size; int level; };
std::vector<Item> items;
core::bounds3d area;
for (MeshData* mesh : group->meshes)
{
auto it = m_tile_size.find(mesh);
if (it == m_tile_size.end() || !(it->second > 0.0) || !mesh->vertex_list ||
mesh->draw_call_list.empty() || !mesh->draw_call_list[0].is_ge_mesh() || !mesh->bbox_ws.b_valid)
continue;
items.push_back({ mesh, it->second, int(floor(log2(it->second) + 0.5)) });
area += mesh->bbox_ws;
}
if (items.size() < 2)
return;
// Stable: within a level, tiles keep the group's order (older captures
// first), so of two copies of a tile the one already shown survives.
std::stable_sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return a.size < b.size; });
// Ground coverage grid (XY, metres), fine enough for the finest tiles.
double cell = items.front().size / 8.0;