df = df.sort_values(by=["x","y"], ascending=False)
df['rank']=tuple(zip(df.x,df.y))
df['rank']=df.groupby('sort_by',sort=False)['rank'].apply(lambda x : pd.Series(pd.factorize(x)[0])).values
df
Out[615]: 
  sort_by      x       y  rank
3       a  500.0  1000.0     1
1       a  200.0  2000.0     2
2       a  200.0  2000.0     2
7       a  200.0   100.5     3
0       a  100.5  4000.0     4
6       b    3.0   600.5     1
5       b    2.0   600.5     2
4       b    1.0   500.5     3