To make this detection really reliable I had to change the RTC register check like this:
Wire.begin(21, 22);
uint8_t reason = M5.RTC.readReg(0x01);
// now it's safe
M5.begin();
// check reboot reason flag: TIE (timer int enable) && TF (timer flag active)
if ((reason & 0b0000101) == 0b0000101) {
restartByRTC = true;
Serial.println("Reboot by RTC");
} else {
restartByRTC = false;
Serial.println("Reboot by power button / USB");
}