// Overlay-based components have a panelClass property (or similar) 
// that can be used to target the overlay pane.

this.dialog.open(DialogExampleComponent, { 
    panelClass: 'dialog-container-custom' 
});

// then style it from there

.dialog-container-custom .mat-dialog-container {
    padding: 0;
    background: grey;
}